Notebooks/05_BART-Illustration.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "38ea58628b274b5a92d89a1f210fe82a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0b1a60e80ecd49aebd1530aa0a54bd01",
              "IPY_MODEL_df71238a03a64fc2b07c03a0b9b0590a",
              "IPY_MODEL_2b99070c1ab04033952bb02552f2c9cd"
            ],
            "layout": "IPY_MODEL_cc968ce980404158bd6772c25d3ad861"
          }
        },
        "0b1a60e80ecd49aebd1530aa0a54bd01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b392b755cb3438394c58eb8033fd90d",
            "placeholder": "​",
            "style": "IPY_MODEL_148b8730310e47199c911dac643293d4",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "df71238a03a64fc2b07c03a0b9b0590a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87f63bd5a8ac489cad69b38c8938429d",
            "max": 26,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4124aa94a035458592f15c9c768cbef2",
            "value": 26
          }
        },
        "2b99070c1ab04033952bb02552f2c9cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81914f515afd4174a20b8bd9a59888f4",
            "placeholder": "​",
            "style": "IPY_MODEL_bd00de78567d4e299acb4c8bf98ade80",
            "value": " 26.0/26.0 [00:00&lt;00:00, 424B/s]"
          }
        },
        "cc968ce980404158bd6772c25d3ad861": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b392b755cb3438394c58eb8033fd90d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "148b8730310e47199c911dac643293d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87f63bd5a8ac489cad69b38c8938429d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4124aa94a035458592f15c9c768cbef2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "81914f515afd4174a20b8bd9a59888f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd00de78567d4e299acb4c8bf98ade80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52a6705b5e91431d84fddecc1213d7d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a907284b1ba4fb280043d05c176a097",
              "IPY_MODEL_cca2fe504e374b1a961fb34b308a6f55",
              "IPY_MODEL_2b0d7659c6bf455ba4e6d7d8f9caa27d"
            ],
            "layout": "IPY_MODEL_bef6f89415b641acb378b3203e4f0288"
          }
        },
        "7a907284b1ba4fb280043d05c176a097": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e663cb0b862a4545bfe2ac4b533b4802",
            "placeholder": "​",
            "style": "IPY_MODEL_d609d4af852f42e4959b9b491afb3cf9",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "cca2fe504e374b1a961fb34b308a6f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9404001969c4c8aab3f7f881e82c561",
            "max": 1432,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d689dee4281f44a583e9288d45a4ad4e",
            "value": 1432
          }
        },
        "2b0d7659c6bf455ba4e6d7d8f9caa27d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a4bf070dd1ed480f99f1ed828bbeaced",
            "placeholder": "​",
            "style": "IPY_MODEL_aa6fe8018f1741699b5b4c5721c56e0e",
            "value": " 1.43k/1.43k [00:00&lt;00:00, 38.8kB/s]"
          }
        },
        "bef6f89415b641acb378b3203e4f0288": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e663cb0b862a4545bfe2ac4b533b4802": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d609d4af852f42e4959b9b491afb3cf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e9404001969c4c8aab3f7f881e82c561": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d689dee4281f44a583e9288d45a4ad4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a4bf070dd1ed480f99f1ed828bbeaced": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa6fe8018f1741699b5b4c5721c56e0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "93fb8be36b904369bbc8ea3277206dbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_293b52e6d2a9434d8da12fbe8f825978",
              "IPY_MODEL_9d9973ae4b224931a1e2fe07ecbb48d7",
              "IPY_MODEL_21e3eeeef41046c5a26947d401f8495f"
            ],
            "layout": "IPY_MODEL_fe6cac73beaa439cb7932df8792fc55d"
          }
        },
        "293b52e6d2a9434d8da12fbe8f825978": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58a7d30d963b47828a64aa2af353284e",
            "placeholder": "​",
            "style": "IPY_MODEL_279c445a94074668929f04869e8b9966",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "9d9973ae4b224931a1e2fe07ecbb48d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8dd6fd7004a6422383bad88b237d47a2",
            "max": 898822,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ec4268d7e4644131ade5f658c173a029",
            "value": 898822
          }
        },
        "21e3eeeef41046c5a26947d401f8495f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ecddebdd1cd045eeb07593a95b084bd8",
            "placeholder": "​",
            "style": "IPY_MODEL_a70ea74660a34b43abd1bbe3680ef8e8",
            "value": " 899k/899k [00:00&lt;00:00, 5.97MB/s]"
          }
        },
        "fe6cac73beaa439cb7932df8792fc55d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58a7d30d963b47828a64aa2af353284e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "279c445a94074668929f04869e8b9966": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8dd6fd7004a6422383bad88b237d47a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec4268d7e4644131ade5f658c173a029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ecddebdd1cd045eeb07593a95b084bd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a70ea74660a34b43abd1bbe3680ef8e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6228ba911fb04bf881ddc75abfa04714": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cd1fb133f98f4894906b4f56e49d51e7",
              "IPY_MODEL_4cb9687d6fdd460aae22271e07a98293",
              "IPY_MODEL_7dfbd522de434b09ae3158d5a1e865ee"
            ],
            "layout": "IPY_MODEL_6e335315dd2040aa8a87297111e2cc0f"
          }
        },
        "cd1fb133f98f4894906b4f56e49d51e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea9a567fe48545b38d16484ef9d153bd",
            "placeholder": "​",
            "style": "IPY_MODEL_8dfebc94278f4688bbd0c89cd7a68355",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "4cb9687d6fdd460aae22271e07a98293": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9e152dcac524381b22d8648195176ee",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_90c65098059b473eac49f7395783f442",
            "value": 456318
          }
        },
        "7dfbd522de434b09ae3158d5a1e865ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5389f523f40e40a9b5135d715f63734a",
            "placeholder": "​",
            "style": "IPY_MODEL_a1ba48366b8f46f5bc5e33b539314549",
            "value": " 456k/456k [00:00&lt;00:00, 3.09MB/s]"
          }
        },
        "6e335315dd2040aa8a87297111e2cc0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea9a567fe48545b38d16484ef9d153bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dfebc94278f4688bbd0c89cd7a68355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b9e152dcac524381b22d8648195176ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90c65098059b473eac49f7395783f442": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5389f523f40e40a9b5135d715f63734a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1ba48366b8f46f5bc5e33b539314549": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c5e458d8c43d4860af4df161b4903b1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a3afb38bed24c4ca62f758765429fca",
              "IPY_MODEL_d692182ef5144965ade733cdb0174e1c",
              "IPY_MODEL_d6138f5c6a5746bda6bfc513dff0a402"
            ],
            "layout": "IPY_MODEL_a4307ec3894547c794856f531aad0ce0"
          }
        },
        "2a3afb38bed24c4ca62f758765429fca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06008c5365ec497dbb197bd72ac1b42f",
            "placeholder": "​",
            "style": "IPY_MODEL_cfbd8358011d44d28c59a0daf313f987",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "d692182ef5144965ade733cdb0174e1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_accf9b508365438d900244da5a2a21f7",
            "max": 150,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_faa6fdc940284bdeb4e94619f182b93b",
            "value": 150
          }
        },
        "d6138f5c6a5746bda6bfc513dff0a402": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5620011674f45c09019a03337aac7fb",
            "placeholder": "​",
            "style": "IPY_MODEL_25abc4c3899e4d9f972d3b334d09d725",
            "value": " 150/150 [00:00&lt;00:00, 2.75kB/s]"
          }
        },
        "a4307ec3894547c794856f531aad0ce0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06008c5365ec497dbb197bd72ac1b42f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfbd8358011d44d28c59a0daf313f987": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "accf9b508365438d900244da5a2a21f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "faa6fdc940284bdeb4e94619f182b93b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c5620011674f45c09019a03337aac7fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25abc4c3899e4d9f972d3b334d09d725": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd6b7e27bd1e45ad9a2ac259be90b5b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7b5ce40a3724c62bc933d0d0054c0a3",
              "IPY_MODEL_ba831d703d04483da4e4480bf87e5fd6",
              "IPY_MODEL_9c9fb8cc2e5a4e9f80248aef5d834bdc"
            ],
            "layout": "IPY_MODEL_ef10fb5dfc38447daada051f78bab7dd"
          }
        },
        "e7b5ce40a3724c62bc933d0d0054c0a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_07c24ec4ad6849de868f6d98b21d48c5",
            "placeholder": "​",
            "style": "IPY_MODEL_00cb6626b7514f94ad23e60ed04d3019",
            "value": "Downloading (…)&quot;pytorch_model.bin&quot;;: 100%"
          }
        },
        "ba831d703d04483da4e4480bf87e5fd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_831195eee083459a876904c0c4b3b4ad",
            "max": 1625285449,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3488b06c782b4a8d96a6edc70d6636a9",
            "value": 1625285449
          }
        },
        "9c9fb8cc2e5a4e9f80248aef5d834bdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f50b6dc2062c46b0ba27a9f2ab1493ac",
            "placeholder": "​",
            "style": "IPY_MODEL_0e0dd23392174f489eb089e95a5bdc30",
            "value": " 1.63G/1.63G [00:44&lt;00:00, 33.1MB/s]"
          }
        },
        "ef10fb5dfc38447daada051f78bab7dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "07c24ec4ad6849de868f6d98b21d48c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00cb6626b7514f94ad23e60ed04d3019": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "831195eee083459a876904c0c4b3b4ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3488b06c782b4a8d96a6edc70d6636a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f50b6dc2062c46b0ba27a9f2ab1493ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e0dd23392174f489eb089e95a5bdc30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers -q"
      ],
      "metadata": {
        "id": "N4VD1kTVBIqW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, BartModel,BartConfig\n",
        "\n",
        "# Initializing a BART facebook/bart-large style configuration weights\n",
        "model = BartModel.from_pretrained(\"facebook/bart-base\")\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-large-cnn\")\n"
      ],
      "metadata": {
        "id": "sXmMHlJRGJJW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "TEXT = (\n",
        "    \"PG&E stated it scheduled the blackouts in response to forecasts for high winds \"\n",
        "    \"amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were \"\n",
        "    \"scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.\"\n",
        ")\n",
        "inputs = tokenizer([TEXT], max_length=1024, return_tensors=\"pt\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jGUGunigMk63",
        "outputId": "f071bea5-5d34-45be-e57b-ff1c92491069"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(inputs.keys())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0yRA8GZOLTI3",
        "outputId": "4301a29d-2528-47f7-f401-7a0740044913"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(inputs['input_ids'].shape)\n",
        "print(inputs['attention_mask'].shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PUlByqwCLv0F",
        "outputId": "8e0785ce-feb0-4fe4-f11a-3a7d7c2b20e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([1, 56])\n",
            "torch.Size([1, 56])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs = model(**inputs,return_dict=True)"
      ],
      "metadata": {
        "id": "vJn-u4eaMo-R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(outputs.keys())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NC727ELjMtGe",
        "outputId": "80ddc974-82fc-4c3c-c207-b16872369d15"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "odict_keys(['last_hidden_state', 'past_key_values', 'encoder_last_hidden_state'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"last hidden state\",outputs.last_hidden_state.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hGvQ4bTQM9Zw",
        "outputId": "67153349-c8a9-439c-a7cd-7c03cea4cf89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "last hidden state torch.Size([1, 56, 768])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"past key\",len(outputs.past_key_values))\n",
        "print(outputs.past_key_values[0][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y6_2cIgmNQc7",
        "outputId": "7d75a9b9-95dc-4bdb-c98e-d0c30420fe0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "past key 6\n",
            "tensor([[[[-0.1147, -0.5220, -0.6727,  ...,  0.3431,  0.0759, -0.0243],\n",
            "          [ 0.0545,  0.3640, -0.7171,  ...,  0.0815,  0.0054,  0.3421],\n",
            "          [ 0.8442, -0.6637, -0.3072,  ..., -0.5735, -0.6820,  0.8552],\n",
            "          ...,\n",
            "          [-1.2454,  0.7658,  1.5817,  ..., -0.3995, -0.1713, -0.0848],\n",
            "          [-2.0342,  0.7014,  0.6181,  ...,  0.0440, -0.1218,  1.0048],\n",
            "          [-1.6544,  0.4801, -0.3073,  ..., -0.2715, -0.1649, -0.2765]],\n",
            "\n",
            "         [[ 0.4247, -0.3899, -0.3139,  ...,  0.5761,  0.1789,  1.3113],\n",
            "          [ 0.2842, -0.3870, -0.3964,  ...,  0.5646,  0.0862,  1.3079],\n",
            "          [-0.8362,  0.2443,  0.4941,  ...,  0.7270,  0.3747, -2.9863],\n",
            "          ...,\n",
            "          [ 0.3557, -1.5609, -2.4382,  ...,  0.9765,  1.8773, -2.5829],\n",
            "          [-0.4403, -2.1208, -0.7422,  ...,  1.4561,  0.9696, -2.9593],\n",
            "          [ 0.8482, -2.7829, -0.3561,  ...,  1.0278, -1.6397, -1.5226]],\n",
            "\n",
            "         [[ 0.2488, -0.7200,  0.5254,  ..., -0.1434, -0.5021,  0.6895],\n",
            "          [ 0.3267, -0.6191,  0.6156,  ..., -0.0449, -0.4194,  0.7115],\n",
            "          [-0.3683,  0.4730,  0.4016,  ..., -1.9499,  0.6165,  0.4426],\n",
            "          ...,\n",
            "          [ 1.4296,  0.1372, -2.0901,  ..., -2.0274, -0.1352, -0.2572],\n",
            "          [ 1.3428, -1.0904, -2.9412,  ..., -0.9871, -2.3028, -1.3896],\n",
            "          [ 0.8649, -2.0814,  1.1080,  ...,  0.7100, -0.8694, -1.3594]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-0.2274, -0.0855, -0.3774,  ..., -0.8612, -1.1205, -1.8190],\n",
            "          [ 0.0473, -0.1292, -0.3163,  ..., -1.0761, -0.8974, -1.8521],\n",
            "          [-0.6442,  1.1307, -0.3430,  ...,  1.1476,  0.2829, -1.0524],\n",
            "          ...,\n",
            "          [-2.3274,  0.7109,  1.7549,  ..., -1.3453,  0.6042,  3.4231],\n",
            "          [-2.9477,  2.8334,  0.2284,  ..., -0.1971, -0.2309,  3.2973],\n",
            "          [-3.5577,  1.5521,  0.4292,  ..., -0.7284,  1.1853,  2.6862]],\n",
            "\n",
            "         [[-1.8059,  0.7841, -2.6778,  ...,  1.8138, -0.6681,  0.2544],\n",
            "          [-1.5170, -0.3241, -1.7216,  ...,  0.9097,  0.4760,  0.4110],\n",
            "          [-0.1923, -1.0990, -0.4658,  ..., -3.9919,  0.9690, -2.0518],\n",
            "          ...,\n",
            "          [ 1.8215, -0.0708,  0.7847,  ..., -3.3925,  1.2834,  0.9661],\n",
            "          [ 1.4300,  1.0478,  0.3346,  ..., -4.3375,  2.6524, -0.5409],\n",
            "          [ 1.9203, -0.4627,  0.3491,  ..., -2.3455,  1.6944, -0.9026]],\n",
            "\n",
            "         [[-1.8905, -1.3374, -0.2002,  ..., -0.3515,  3.3732,  0.9060],\n",
            "          [-1.5089, -1.2653, -0.3086,  ..., -0.5588,  3.2395,  0.7418],\n",
            "          [ 3.8071,  2.3115,  2.7365,  ..., -1.3600,  1.7248, -2.9377],\n",
            "          ...,\n",
            "          [-0.3927,  3.4526,  3.6151,  ..., -1.9583, -2.3328, -4.1539],\n",
            "          [-0.8841,  2.3492,  3.0914,  ..., -2.6662, -0.8144, -2.8158],\n",
            "          [ 2.4551,  2.6718,  0.1616,  ...,  1.4295, -0.7736, -1.4752]]]],\n",
            "       grad_fn=<CloneBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"last encoder hidden state\",outputs.encoder_last_hidden_state.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s4VWdrlsNU_l",
        "outputId": "9514e96a-5a60-4ab2-a7b1-da6154b164a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "last encoder hidden state torch.Size([1, 56, 768])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# BART"
      ],
      "metadata": {
        "id": "UuSq7GwDCoSi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text Generation"
      ],
      "metadata": {
        "id": "y2muqSupCjAy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Summarization"
      ],
      "metadata": {
        "id": "Okh15PQ8DhpV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, BartForConditionalGeneration\n",
        "\n",
        "model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-large-cnn\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-large-cnn\")\n",
        "\n",
        "ARTICLE_TO_SUMMARIZE = (\n",
        "    \"PG&E stated it scheduled the blackouts in response to forecasts for high winds \"\n",
        "    \"amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were \"\n",
        "    \"scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.\"\n",
        ")\n",
        "inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors=\"pt\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "jUhBMxl2DmQt",
        "outputId": "757b7408-6de4-422d-fb8c-5a7455409b1c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-89a8699fd944>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBartModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBartForConditionalGeneration\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"facebook/bart-large-cnn\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"facebook/bart-large-cnn\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   2299\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_sharded\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstate_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2300\u001b[0m                 \u001b[0;31m# Time to load the checkpoint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2301\u001b[0;31m                 \u001b[0mstate_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresolved_archive_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2303\u001b[0m             \u001b[0;31m# set dtype to instantiate the model under:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[0;34m(checkpoint_file)\u001b[0m\n\u001b[1;32m    413\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msafe_load_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, weights_only, **pickle_load_args)\u001b[0m\n\u001b[1;32m    793\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    794\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnpicklingError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mUNSAFE_MESSAGE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 795\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_legacy_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    796\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    797\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_legacy_load\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m   1018\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m         \u001b[0mtyped_storage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1020\u001b[0;31m         typed_storage._storage._set_from_file(\n\u001b[0m\u001b[1;32m   1021\u001b[0m             \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moffset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf_should_read_directly\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1022\u001b[0m             torch._utils._element_size(typed_storage.dtype))\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate Summary\n",
        "summary_ids = model.generate(inputs[\"input_ids\"], num_beams=2, min_length=0, max_length=20)\n",
        "tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "7tPWtuOoDmK9",
        "outputId": "dad2f47d-295c-4539-cb70-c524f25d3e71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Next Word Prediction"
      ],
      "metadata": {
        "id": "CGYu8Ch1DUyn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Masked Word Replacement"
      ],
      "metadata": {
        "id": "2itbAFdrDYw7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MyxJLns26yvF",
        "outputId": "493d3aef-404d-4537-c8d7-4929ff1b1062"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['not', 'good', 'healthy', 'great', 'very']"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "from transformers import AutoTokenizer, BartForConditionalGeneration\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-base\")\n",
        "model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-base\")\n",
        "\n",
        "TXT = \"My friends are <mask> but they eat too many carbs.\"\n",
        "input_ids = tokenizer([TXT], return_tensors=\"pt\")[\"input_ids\"]\n",
        "logits = model(input_ids).logits\n",
        "\n",
        "masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()\n",
        "probs = logits[0, masked_index].softmax(dim=0)\n",
        "values, predictions = probs.topk(5)\n",
        "\n",
        "tokenizer.decode(predictions).split()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text Classification"
      ],
      "metadata": {
        "id": "tinveXBVCrzd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, BartForSequenceClassification\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"valhalla/bart-large-sst2\")\n",
        "model = BartForSequenceClassification.from_pretrained(\"valhalla/bart-large-sst2\")\n",
        "\n",
        "inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\")\n",
        "\n",
        "with torch.no_grad():\n",
        "    logits = model(**inputs).logits\n",
        "\n",
        "predicted_class_id = logits.argmax().item()\n",
        "model.config.id2label[predicted_class_id]\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "ezJeQe_3Cqzl",
        "outputId": "03e4b4bb-35a8-4a1d-fdca-2f0d961dcfd5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'NEGATIVE', '1': 'POSITIVE'}. The number of labels wil be overwritten to 2.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'POSITIVE'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Question Answering"
      ],
      "metadata": {
        "id": "gRA3q4bEErSG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, BartForQuestionAnswering\n",
        "import torch\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"valhalla/bart-large-finetuned-squadv1\")\n",
        "model = BartForQuestionAnswering.from_pretrained(\"valhalla/bart-large-finetuned-squadv1\")\n",
        "\n",
        "question, text = \"Who was Jim Henson?\", \"Jim Henson was a nice puppet\"\n",
        "\n",
        "inputs = tokenizer(question, text, return_tensors=\"pt\")\n",
        "with torch.no_grad():\n",
        "    outputs = model(**inputs)\n",
        "\n",
        "answer_start_index = outputs.start_logits.argmax()\n",
        "answer_end_index = outputs.end_logits.argmax()\n",
        "\n",
        "predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]\n",
        "tokenizer.decode(predict_answer_tokens, skip_special_tokens=True)\n",
        "\n",
        "# target is \"nice puppet\"\n",
        "target_start_index = torch.tensor([14])\n",
        "target_end_index = torch.tensor([15])\n",
        "\n",
        "outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index)\n",
        "loss = outputs.loss\n",
        "round(loss.item(), 2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296,
          "referenced_widgets": [
            "38ea58628b274b5a92d89a1f210fe82a",
            "0b1a60e80ecd49aebd1530aa0a54bd01",
            "df71238a03a64fc2b07c03a0b9b0590a",
            "2b99070c1ab04033952bb02552f2c9cd",
            "cc968ce980404158bd6772c25d3ad861",
            "4b392b755cb3438394c58eb8033fd90d",
            "148b8730310e47199c911dac643293d4",
            "87f63bd5a8ac489cad69b38c8938429d",
            "4124aa94a035458592f15c9c768cbef2",
            "81914f515afd4174a20b8bd9a59888f4",
            "bd00de78567d4e299acb4c8bf98ade80",
            "52a6705b5e91431d84fddecc1213d7d3",
            "7a907284b1ba4fb280043d05c176a097",
            "cca2fe504e374b1a961fb34b308a6f55",
            "2b0d7659c6bf455ba4e6d7d8f9caa27d",
            "bef6f89415b641acb378b3203e4f0288",
            "e663cb0b862a4545bfe2ac4b533b4802",
            "d609d4af852f42e4959b9b491afb3cf9",
            "e9404001969c4c8aab3f7f881e82c561",
            "d689dee4281f44a583e9288d45a4ad4e",
            "a4bf070dd1ed480f99f1ed828bbeaced",
            "aa6fe8018f1741699b5b4c5721c56e0e",
            "93fb8be36b904369bbc8ea3277206dbe",
            "293b52e6d2a9434d8da12fbe8f825978",
            "9d9973ae4b224931a1e2fe07ecbb48d7",
            "21e3eeeef41046c5a26947d401f8495f",
            "fe6cac73beaa439cb7932df8792fc55d",
            "58a7d30d963b47828a64aa2af353284e",
            "279c445a94074668929f04869e8b9966",
            "8dd6fd7004a6422383bad88b237d47a2",
            "ec4268d7e4644131ade5f658c173a029",
            "ecddebdd1cd045eeb07593a95b084bd8",
            "a70ea74660a34b43abd1bbe3680ef8e8",
            "6228ba911fb04bf881ddc75abfa04714",
            "cd1fb133f98f4894906b4f56e49d51e7",
            "4cb9687d6fdd460aae22271e07a98293",
            "7dfbd522de434b09ae3158d5a1e865ee",
            "6e335315dd2040aa8a87297111e2cc0f",
            "ea9a567fe48545b38d16484ef9d153bd",
            "8dfebc94278f4688bbd0c89cd7a68355",
            "b9e152dcac524381b22d8648195176ee",
            "90c65098059b473eac49f7395783f442",
            "5389f523f40e40a9b5135d715f63734a",
            "a1ba48366b8f46f5bc5e33b539314549",
            "c5e458d8c43d4860af4df161b4903b1a",
            "2a3afb38bed24c4ca62f758765429fca",
            "d692182ef5144965ade733cdb0174e1c",
            "d6138f5c6a5746bda6bfc513dff0a402",
            "a4307ec3894547c794856f531aad0ce0",
            "06008c5365ec497dbb197bd72ac1b42f",
            "cfbd8358011d44d28c59a0daf313f987",
            "accf9b508365438d900244da5a2a21f7",
            "faa6fdc940284bdeb4e94619f182b93b",
            "c5620011674f45c09019a03337aac7fb",
            "25abc4c3899e4d9f972d3b334d09d725",
            "bd6b7e27bd1e45ad9a2ac259be90b5b1",
            "e7b5ce40a3724c62bc933d0d0054c0a3",
            "ba831d703d04483da4e4480bf87e5fd6",
            "9c9fb8cc2e5a4e9f80248aef5d834bdc",
            "ef10fb5dfc38447daada051f78bab7dd",
            "07c24ec4ad6849de868f6d98b21d48c5",
            "00cb6626b7514f94ad23e60ed04d3019",
            "831195eee083459a876904c0c4b3b4ad",
            "3488b06c782b4a8d96a6edc70d6636a9",
            "f50b6dc2062c46b0ba27a9f2ab1493ac",
            "0e0dd23392174f489eb089e95a5bdc30"
          ]
        },
        "id": "3jCA0B9KEnRd",
        "outputId": "7f3a9218-8e03-413d-d13f-cd4275ef7497"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "38ea58628b274b5a92d89a1f210fe82a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.43k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "52a6705b5e91431d84fddecc1213d7d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'LABEL_0', '1': 'LABEL_1'}. The number of labels wil be overwritten to 2.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "93fb8be36b904369bbc8ea3277206dbe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6228ba911fb04bf881ddc75abfa04714"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/150 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c5e458d8c43d4860af4df161b4903b1a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'LABEL_0', '1': 'LABEL_1'}. The number of labels wil be overwritten to 2.\n",
            "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'LABEL_0', '1': 'LABEL_1'}. The number of labels wil be overwritten to 2.\n",
            "You passed along `num_labels=3` with an incompatible id to label map: {'0': 'LABEL_0', '1': 'LABEL_1'}. The number of labels wil be overwritten to 2.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/1.63G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd6b7e27bd1e45ad9a2ac259be90b5b1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.59"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Other"
      ],
      "metadata": {
        "id": "8QLsj7jPFaF0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, BartForCausalLM\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-base\")\n",
        "model = BartForCausalLM.from_pretrained(\"facebook/bart-base\", add_cross_attention=False)\n",
        "assert model.config.is_decoder, f\"{model.__class__} has to be configured as a decoder.\"\n",
        "inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\")\n",
        "outputs = model(**inputs)\n",
        "\n",
        "logits = outputs.logits\n",
        "expected_shape = [1, inputs.input_ids.shape[-1], model.config.vocab_size]\n",
        "list(logits.shape) == expected_shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CAsgpkNkFXY5",
        "outputId": "b65aa577-ccd9-48e1-de37-8a537bee187a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at facebook/bart-base were not used when initializing BartForCausalLM: ['encoder.layers.3.fc1.bias', 'encoder.layers.3.fc1.weight', 'encoder.layers.1.self_attn.v_proj.weight', 'encoder.layers.2.self_attn.q_proj.weight', 'shared.weight', 'encoder.layers.5.self_attn.v_proj.bias', 'encoder.layers.0.self_attn.v_proj.bias', 'encoder.layers.3.self_attn.out_proj.weight', 'encoder.layers.4.self_attn_layer_norm.bias', 'encoder.layers.5.self_attn.v_proj.weight', 'encoder.layers.1.self_attn.k_proj.bias', 'encoder.layers.1.self_attn.out_proj.bias', 'encoder.layers.3.self_attn.v_proj.bias', 'encoder.layers.0.self_attn.k_proj.weight', 'encoder.layers.5.self_attn.q_proj.weight', 'encoder.layers.1.fc1.weight', 'encoder.layers.0.self_attn.out_proj.bias', 'encoder.layers.1.self_attn.out_proj.weight', 'encoder.layers.4.self_attn.q_proj.weight', 'encoder.layers.5.fc1.weight', 'encoder.layers.2.self_attn.q_proj.bias', 'encoder.embed_tokens.weight', 'encoder.layers.2.fc1.bias', 'encoder.layers.1.fc2.weight', 'encoder.layers.5.self_attn.out_proj.bias', 'encoder.layers.4.self_attn_layer_norm.weight', 'encoder.layers.3.self_attn_layer_norm.weight', 'encoder.layers.4.fc2.weight', 'encoder.layers.5.final_layer_norm.weight', 'encoder.layers.2.fc1.weight', 'encoder.layers.5.self_attn_layer_norm.bias', 'encoder.layers.2.self_attn.out_proj.weight', 'encoder.layers.1.fc1.bias', 'encoder.layers.4.fc2.bias', 'encoder.layers.0.fc1.weight', 'encoder.layers.2.self_attn_layer_norm.weight', 'encoder.layers.0.self_attn.q_proj.bias', 'encoder.layers.5.self_attn.k_proj.bias', 'encoder.layers.4.self_attn.out_proj.weight', 'encoder.layers.5.final_layer_norm.bias', 'encoder.layers.3.self_attn.k_proj.weight', 'encoder.layers.2.fc2.bias', 'encoder.layers.3.self_attn.out_proj.bias', 'encoder.layers.0.fc2.weight', 'encoder.layers.0.final_layer_norm.bias', 'encoder.layers.4.self_attn.v_proj.bias', 'encoder.layers.3.self_attn.v_proj.weight', 'encoder.layers.4.self_attn.out_proj.bias', 'encoder.layers.2.final_layer_norm.weight', 'encoder.layers.0.final_layer_norm.weight', 'encoder.layers.5.self_attn_layer_norm.weight', 'encoder.layers.0.self_attn.v_proj.weight', 'encoder.layers.3.self_attn_layer_norm.bias', 'encoder.layers.5.self_attn.q_proj.bias', 'encoder.layers.0.fc1.bias', 'encoder.layernorm_embedding.weight', 'encoder.layers.4.fc1.weight', 'encoder.embed_positions.weight', 'encoder.layers.3.final_layer_norm.bias', 'encoder.layers.1.self_attn_layer_norm.bias', 'encoder.layers.2.self_attn.v_proj.weight', 'encoder.layernorm_embedding.bias', 'encoder.layers.4.fc1.bias', 'encoder.layers.2.self_attn.v_proj.bias', 'encoder.layers.0.self_attn_layer_norm.bias', 'encoder.layers.4.self_attn.v_proj.weight', 'encoder.layers.4.self_attn.k_proj.weight', 'encoder.layers.2.self_attn.out_proj.bias', 'encoder.layers.1.fc2.bias', 'encoder.layers.0.self_attn.k_proj.bias', 'encoder.layers.3.final_layer_norm.weight', 'encoder.layers.4.self_attn.q_proj.bias', 'encoder.layers.1.self_attn.k_proj.weight', 'encoder.layers.0.self_attn.q_proj.weight', 'encoder.layers.1.self_attn.v_proj.bias', 'encoder.layers.0.fc2.bias', 'encoder.layers.3.self_attn.q_proj.bias', 'encoder.layers.1.self_attn.q_proj.bias', 'encoder.layers.5.fc1.bias', 'encoder.layers.2.self_attn.k_proj.bias', 'encoder.layers.0.self_attn_layer_norm.weight', 'encoder.layers.2.self_attn_layer_norm.bias', 'encoder.layers.5.self_attn.k_proj.weight', 'encoder.layers.4.self_attn.k_proj.bias', 'encoder.layers.3.self_attn.k_proj.bias', 'encoder.layers.4.final_layer_norm.bias', 'encoder.layers.2.fc2.weight', 'encoder.layers.4.final_layer_norm.weight', 'encoder.layers.2.self_attn.k_proj.weight', 'encoder.layers.3.fc2.weight', 'encoder.layers.3.self_attn.q_proj.weight', 'encoder.layers.1.final_layer_norm.weight', 'encoder.layers.2.final_layer_norm.bias', 'encoder.layers.5.fc2.weight', 'encoder.layers.1.self_attn.q_proj.weight', 'encoder.layers.5.fc2.bias', 'encoder.layers.1.self_attn_layer_norm.weight', 'encoder.layers.1.final_layer_norm.bias', 'encoder.layers.0.self_attn.out_proj.weight', 'encoder.layers.5.self_attn.out_proj.weight', 'encoder.layers.3.fc2.bias']\n",
            "- This IS expected if you are initializing BartForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BartForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    }
  ]
}