BERT Word Embeddings.ipynb

bert-word-embeddings.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/firmai/44fe3b381f82abdbe0420b6c7a27ee3a/bert-word-embeddings.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R7RkTtfLAiEh"
      },
      "source": [
        "# BERT Word Embeddings\n",
        "\n",
        "Three ways to obtain word embeddings from BERT:\n",
        "- context-free\n",
        "- context-based\n",
        "- context-averaged\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Laz0oC4qAiEk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "31c8e7d8-6f81-4f59-a157-d1a915fc4201"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (2.0.0+cu118)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from torch) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from torch) (3.11.0)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.9/dist-packages (from torch) (2.0.0)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (16.0.1)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (3.25.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->torch) (2.1.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->torch) (1.3.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.13.4-py3-none-any.whl (200 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.1/200.1 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.13.4 tokenizers-0.13.3 transformers-4.28.1\n"
          ]
        }
      ],
      "source": [
        "!pip install torch\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "A_UT8pFuAiEl"
      },
      "outputs": [],
      "source": [
        "from transformers import BertTokenizer, BertModel\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can use finbert, but in my opinion, it doesn't pperform that great (if used change hidden_states = outputs[2][1:] to hidden_states = outputs[1][1:] )\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"ProsusAI/finbert\")\n",
        "# model = AutoModelForSequenceClassification.from_pretrained(\"ProsusAI/finbert\",output_hidden_states = True)"
      ],
      "metadata": {
        "id": "1pCZxhw-DcOR"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "603f391aad42446980c41f1a34ce3481",
            "e805150389f546c7ada90adf1cfc8006",
            "688dcd76bd434b29a17ca62eabdbc2c0",
            "52b76b32261f4766b22c733f4a0f3abe",
            "e90987d9d8864ac9a52b14a9076d4121",
            "8d09cb09150b4802bcb22668636bce1b",
            "7ddd3144dfaf4c8ea5838d4767581088",
            "684e1618fde84e58b9546cfceaa54bc6",
            "d7239dcd186d4aa7b7b893233ef75624",
            "8a0716c071ea402dbf375a65b1a2f2bd",
            "97810075215742c69afcd77d0651d40a",
            "8540b0a6159e4d6eb64e18aa0504233a",
            "28ae02c0458640f0a1491c97faaa5715",
            "93f20db69e2b449aaef50fd108ea8578",
            "8792c1eb666b46748fbdb0baaedcb2b1",
            "718b9ca932af473e8ce793bea0197f10",
            "4fdcfd3da5c44a289c218aa0942a1c5a",
            "8871cb97f963455ab328489da803d0e9",
            "0aad6377798f466283bba6af0c367d46",
            "193855bc315b4922b232db772dec7452",
            "42389b2495cf44a29f833aed7114d0f0",
            "98aadd6045694768a25814f629ffda89",
            "bd0a9fa92d4442c0bb21cae97e5b1538",
            "d976b5f1e2034b5ea2bda4dc877e1d3d",
            "f3ded921ee124b02920419695181173d",
            "dd329ca3ac8a411793c1a6b863f751ed",
            "7aa436546c4e4bf2b270f7fec6e91590",
            "2defd2a76be94f84ba957b21477909c1",
            "80f82c5f6cfe494fa56eef70dfcdd05c",
            "e4a2612024dd45a981ed7392a7513470",
            "8e1f8f179f6e40f7978b005419b2929e",
            "800b1520c0ac4bc3b43074ab21e8a5f4",
            "3580f7c0b9b04ac7b9e32e5c2a9989a6",
            "65c4004f46f74db38d53e9c611e33879",
            "4829b7f750314cff996cb820e559dc2c",
            "fc4b069dc76a46f9ab0ea6fc6656c2ea",
            "8d3bdcd379ef43f181e1814d41b4d95f",
            "384c4c1f7752480cae2af7c28c854dab",
            "8bed55b4fea84dcf92a6401d34cb0d67",
            "08999f81dc6446ae9adba574dd7ee444",
            "f5aa090da8fd4e0ab0dcd6cd6de1f512",
            "1aebcc0db0a1491b94f64a34fa62fc68",
            "67fe0f157fec443693c94a40e78515c2",
            "f5bdbc4307414c71956af3baa51a0261"
          ],
          "base_uri": "https://localhost:8080/",
          "height": 246
        },
        "id": "CxICRSJeAiEm",
        "outputId": "5fb7b5b0-c3cb-4db1-e8c7-b499ee2fdd15"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "603f391aad42446980c41f1a34ce3481"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8540b0a6159e4d6eb64e18aa0504233a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd0a9fa92d4442c0bb21cae97e5b1538"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "65c4004f46f74db38d53e9c611e33879"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Loading the pre-trained BERT model\n",
        "###################################\n",
        "# Embeddings will be derived from\n",
        "# the outputs of this model\n",
        "model = BertModel.from_pretrained('bert-base-uncased',\n",
        "                                  output_hidden_states = True,\n",
        "                                  )\n",
        "\n",
        "# Setting up the tokenizer\n",
        "###################################\n",
        "# This is the same tokenizer that\n",
        "# was used in the model to generate \n",
        "# embeddings to ensure consistency\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Extracting word embeddings: We are going to generate embeddings for the following texts"
      ],
      "metadata": {
        "id": "Tz5K35GRGWR9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ejn8K57zAiEn"
      },
      "outputs": [],
      "source": [
        "# Text corpus\n",
        "##############\n",
        "# These sentences show the different\n",
        "# forms of the word 'bank' to show the\n",
        "# value of contextualized embeddings\n",
        "\n",
        "texts = [\"bank\",\n",
        "         \"The river bank was flooded.\",\n",
        "         \"The bank vault was robust.\",\n",
        "         \"He had to bank on her for support.\",\n",
        "         \"The bank was out of money.\",\n",
        "         \"The bank teller was a man.\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We also need some functions to massage the input into the right form"
      ],
      "metadata": {
        "id": "HsTAuFDRGM2f"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "xG4vdR38AiEo"
      },
      "outputs": [],
      "source": [
        "def bert_text_preparation(text, tokenizer):\n",
        "    \"\"\"Preparing the input for BERT\n",
        "    \n",
        "    Takes a string argument and performs\n",
        "    pre-processing like adding special tokens,\n",
        "    tokenization, tokens to ids, and tokens to\n",
        "    segment ids. All tokens are mapped to seg-\n",
        "    ment id = 1.\n",
        "    \n",
        "    Args:\n",
        "        text (str): Text to be converted\n",
        "        tokenizer (obj): Tokenizer object\n",
        "            to convert text into BERT-re-\n",
        "            adable tokens and ids\n",
        "        \n",
        "    Returns:\n",
        "        list: List of BERT-readable tokens\n",
        "        obj: Torch tensor with token ids\n",
        "        obj: Torch tensor segment ids\n",
        "    \n",
        "    \n",
        "    \"\"\"\n",
        "    marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
        "    tokenized_text = tokenizer.tokenize(marked_text)\n",
        "    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "    segments_ids = [1]*len(indexed_tokens)\n",
        "\n",
        "    # Convert inputs to PyTorch tensors\n",
        "    tokens_tensor = torch.tensor([indexed_tokens])\n",
        "    segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "    return tokenized_text, tokens_tensor, segments_tensors\n",
        "    \n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "And another function to convert the input into embeddings"
      ],
      "metadata": {
        "id": "ggAavnmdGPpm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_bert_embeddings(tokens_tensor, segments_tensors, model):\n",
        "    \"\"\"Get embeddings from an embedding model\n",
        "    \n",
        "    Args:\n",
        "        tokens_tensor (obj): Torch tensor size [n_tokens]\n",
        "            with token ids for each token in text\n",
        "        segments_tensors (obj): Torch tensor size [n_tokens]\n",
        "            with segment ids for each token in text\n",
        "        model (obj): Embedding model to generate embeddings\n",
        "            from token and segment ids\n",
        "    \n",
        "    Returns:\n",
        "        list: List of list of floats of size\n",
        "            [n_tokens, n_embedding_dimensions]\n",
        "            containing embeddings for each token\n",
        "    \n",
        "    \"\"\"\n",
        "    \n",
        "    # Gradient calculation id disabled\n",
        "    # Model is in inference mode\n",
        "    with torch.no_grad():\n",
        "        outputs = model(tokens_tensor, segments_tensors)\n",
        "        # Removing the first hidden state\n",
        "        # The first state is the input state\n",
        "        hidden_states = outputs[2][1:]\n",
        "\n",
        "    # Getting embeddings from the final BERT layer\n",
        "    token_embeddings = hidden_states[-1]\n",
        "    # Collapsing the tensor into 1-dimension\n",
        "    token_embeddings = torch.squeeze(token_embeddings, dim=0)\n",
        "    # Converting torchtensors to lists\n",
        "    list_token_embeddings = [token_embed.tolist() for token_embed in token_embeddings]\n",
        "\n",
        "    return list_token_embeddings"
      ],
      "metadata": {
        "id": "Cw0T1OW8GQcd"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Embeddings are generated in the following manner"
      ],
      "metadata": {
        "id": "PyZ-TNtYGaUS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "M4x1IHYcAiEo"
      },
      "outputs": [],
      "source": [
        "# Getting embeddings for the target\n",
        "# word in all given contexts\n",
        "target_word_embeddings = []\n",
        "\n",
        "for text in texts:\n",
        "    tokenized_text, tokens_tensor, segments_tensors = bert_text_preparation(text, tokenizer)\n",
        "    list_token_embeddings = get_bert_embeddings(tokens_tensor, segments_tensors, model)\n",
        "    \n",
        "    # Find the position 'bank' in list of tokens\n",
        "    word_index = tokenized_text.index('bank')\n",
        "    # Get the embedding for bank\n",
        "    word_embedding = list_token_embeddings[word_index]\n",
        "\n",
        "    target_word_embeddings.append(word_embedding)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finally, distances between the embeddings for the word bank in different contexts are calculated using this code, we create a Pandas DataFrame to store all the distances."
      ],
      "metadata": {
        "id": "LazQhpYrGeIg"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "4A7jHL0RAiEp"
      },
      "outputs": [],
      "source": [
        "from scipy.spatial.distance import cosine\n",
        "\n",
        "# Calculating the distance between the\n",
        "# embeddings of 'bank' in all the\n",
        "# given contexts of the word\n",
        "\n",
        "list_of_distances = []\n",
        "for text1, embed1 in zip(texts, target_word_embeddings):\n",
        "    for text2, embed2 in zip(texts, target_word_embeddings):\n",
        "        cos_dist = 1 - cosine(embed1, embed2)\n",
        "        list_of_distances.append([text1, text2, cos_dist])\n",
        "\n",
        "distances_df = pd.DataFrame(list_of_distances, columns=['text1', 'text2', 'similarity'])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### “Context-free” pre-trained embeddings\n",
        "\n",
        "The first text (\"bank\") generates a context-free text embedding. This is context-free since there are no accompanying words to provide context to the meaning of \"bank\". In a way, this is the average across all embeddings of the word \"bank\".\n",
        "Understandably, this context-free embedding does not look like one usage of the word \"bank\". This is evident in the cosine distance between the context-free embedding and all other versions of the word."
      ],
      "metadata": {
        "id": "MZjDG_03G0A7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "embed2"
      ],
      "metadata": {
        "id": "nFEDztPNqQ4q",
        "outputId": "d8890aa4-91b4-42ca-b42b-cc237d5fc979",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.7716087698936462,\n",
              " -0.40309032797813416,\n",
              " -0.29921749234199524,\n",
              " 0.052665021270513535,\n",
              " 0.36171749234199524,\n",
              " 0.7383619546890259,\n",
              " -0.32540130615234375,\n",
              " 0.5796908736228943,\n",
              " 0.1448058933019638,\n",
              " -0.783697783946991,\n",
              " 0.32631152868270874,\n",
              " -0.033591244369745255,\n",
              " 0.07557772099971771,\n",
              " -0.18161912262439728,\n",
              " -0.4847656786441803,\n",
              " 0.1046987771987915,\n",
              " 0.5212904810905457,\n",
              " 0.4489266574382782,\n",
              " 0.40076857805252075,\n",
              " 0.023557813838124275,\n",
              " -1.0827289819717407,\n",
              " -0.189421609044075,\n",
              " 0.46014147996902466,\n",
              " 0.22436052560806274,\n",
              " 0.08412308990955353,\n",
              " 0.3808949291706085,\n",
              " 0.40088602900505066,\n",
              " 0.34641796350479126,\n",
              " -0.10983875393867493,\n",
              " -0.05407490208745003,\n",
              " 0.4438695013523102,\n",
              " 0.5756655335426331,\n",
              " 0.448840469121933,\n",
              " 0.3600899875164032,\n",
              " -0.20543277263641357,\n",
              " 0.07272572815418243,\n",
              " -0.283610463142395,\n",
              " 0.11001996695995331,\n",
              " -0.9104434251785278,\n",
              " 0.029562612995505333,\n",
              " -0.24418990314006805,\n",
              " -0.7000628709793091,\n",
              " -0.06389942020177841,\n",
              " 0.04888778552412987,\n",
              " -0.3355029821395874,\n",
              " 0.22565120458602905,\n",
              " -0.18307848274707794,\n",
              " 0.34762096405029297,\n",
              " -0.9771680235862732,\n",
              " -0.25422683358192444,\n",
              " -0.30704328417778015,\n",
              " 0.8428120613098145,\n",
              " -0.09705250710248947,\n",
              " -0.47677671909332275,\n",
              " 0.19244298338890076,\n",
              " 0.7679344415664673,\n",
              " -0.7784724235534668,\n",
              " -0.4639985263347626,\n",
              " 0.30759912729263306,\n",
              " -0.5025826096534729,\n",
              " 0.7814780473709106,\n",
              " 0.34427663683891296,\n",
              " 0.5240030288696289,\n",
              " -0.9884239435195923,\n",
              " -0.09247169643640518,\n",
              " -0.24887126684188843,\n",
              " 0.5101141333580017,\n",
              " -0.2979409396648407,\n",
              " 0.1586844027042389,\n",
              " 0.09471500664949417,\n",
              " -0.25339508056640625,\n",
              " 1.1324656009674072,\n",
              " -0.5025771856307983,\n",
              " -1.0372943878173828,\n",
              " -0.061141062527894974,\n",
              " 0.13247205317020416,\n",
              " 0.12954936921596527,\n",
              " 0.07911813259124756,\n",
              " 0.30207428336143494,\n",
              " -0.23245571553707123,\n",
              " -0.11368819326162338,\n",
              " -0.4897908568382263,\n",
              " 0.26160672307014465,\n",
              " 0.26042604446411133,\n",
              " 0.3771438002586365,\n",
              " -0.2295927107334137,\n",
              " -0.3804225027561188,\n",
              " 0.06993488222360611,\n",
              " 0.3466036319732666,\n",
              " 0.45849958062171936,\n",
              " -0.7176180481910706,\n",
              " 0.4275922179222107,\n",
              " -0.45328569412231445,\n",
              " 0.2647191882133484,\n",
              " -0.8459752202033997,\n",
              " 0.1247115358710289,\n",
              " -0.13423427939414978,\n",
              " 0.0916450098156929,\n",
              " 0.09430268406867981,\n",
              " 0.14447656273841858,\n",
              " 0.17516019940376282,\n",
              " -0.8052711486816406,\n",
              " 0.3388497531414032,\n",
              " 0.12579400837421417,\n",
              " -0.3774828314781189,\n",
              " -0.253370463848114,\n",
              " -0.5764696598052979,\n",
              " -0.5694563984870911,\n",
              " -0.026075035333633423,\n",
              " 0.10481689125299454,\n",
              " 0.5834141373634338,\n",
              " -0.3299909234046936,\n",
              " 0.45716893672943115,\n",
              " -0.44311976432800293,\n",
              " -0.07554926723241806,\n",
              " 0.22576038539409637,\n",
              " 0.1947816014289856,\n",
              " -0.8227598667144775,\n",
              " -0.3799953758716583,\n",
              " -0.468389630317688,\n",
              " 0.29161933064460754,\n",
              " 0.9331499338150024,\n",
              " 0.41290804743766785,\n",
              " 0.8080536127090454,\n",
              " 0.4097137153148651,\n",
              " 0.1095564141869545,\n",
              " -0.2816024124622345,\n",
              " 0.7313909530639648,\n",
              " -0.4790996015071869,\n",
              " -0.331728458404541,\n",
              " 0.3227800130844116,\n",
              " 0.6442036628723145,\n",
              " -0.17978370189666748,\n",
              " -0.6313384771347046,\n",
              " 0.7451621890068054,\n",
              " -0.44734707474708557,\n",
              " -0.3652609884738922,\n",
              " -0.5225532650947571,\n",
              " -0.1918911635875702,\n",
              " 0.6922026872634888,\n",
              " 0.4960022568702698,\n",
              " 0.7192777395248413,\n",
              " 0.14021727442741394,\n",
              " 0.8468748331069946,\n",
              " 0.16828672587871552,\n",
              " 0.6877663731575012,\n",
              " -0.14881286025047302,\n",
              " -0.07684887200593948,\n",
              " -0.2665281593799591,\n",
              " 0.008550762198865414,\n",
              " 0.1127418652176857,\n",
              " -0.04604152962565422,\n",
              " -0.2569620907306671,\n",
              " -0.4100417196750641,\n",
              " 0.44537854194641113,\n",
              " -0.2697429358959198,\n",
              " 0.0026064058765769005,\n",
              " -0.06810358166694641,\n",
              " -0.6113461852073669,\n",
              " -0.05183754861354828,\n",
              " 0.29507970809936523,\n",
              " 0.0995282381772995,\n",
              " 0.046957310289144516,\n",
              " 0.02385854348540306,\n",
              " -0.7850028276443481,\n",
              " -0.7532016038894653,\n",
              " 0.17616990208625793,\n",
              " 0.23652206361293793,\n",
              " -0.19257915019989014,\n",
              " -0.18799301981925964,\n",
              " -0.7664093971252441,\n",
              " -0.3840271532535553,\n",
              " 0.6072332859039307,\n",
              " 0.014171562157571316,\n",
              " 0.9538608193397522,\n",
              " 0.27736467123031616,\n",
              " -0.7593997716903687,\n",
              " -0.5560435652732849,\n",
              " 0.10334096848964691,\n",
              " 0.04470081254839897,\n",
              " 0.0264885313808918,\n",
              " 0.6660721898078918,\n",
              " 0.822999119758606,\n",
              " 1.214438796043396,\n",
              " 0.5543962121009827,\n",
              " -0.2892921566963196,\n",
              " 1.112549066543579,\n",
              " 0.2684617340564728,\n",
              " 0.3573964238166809,\n",
              " 0.24116159975528717,\n",
              " 0.007226062938570976,\n",
              " -0.7152822613716125,\n",
              " -0.21177706122398376,\n",
              " -0.057168446481227875,\n",
              " -0.49640005826950073,\n",
              " -0.39642542600631714,\n",
              " -0.251030296087265,\n",
              " -0.13356545567512512,\n",
              " -0.7973623871803284,\n",
              " -0.2325553447008133,\n",
              " -0.2782469689846039,\n",
              " -0.01265036966651678,\n",
              " 0.4257214665412903,\n",
              " 0.24585239589214325,\n",
              " -0.07413879781961441,\n",
              " 0.5921086072921753,\n",
              " -0.09990530461072922,\n",
              " -0.18087804317474365,\n",
              " 0.853805422782898,\n",
              " 0.1372002214193344,\n",
              " -0.16952551901340485,\n",
              " 0.047132302075624466,\n",
              " 0.020439857617020607,\n",
              " 0.26597633957862854,\n",
              " -0.38421332836151123,\n",
              " 0.5102985501289368,\n",
              " 0.13281658291816711,\n",
              " -0.4930082857608795,\n",
              " -0.10949187725782394,\n",
              " -0.15832382440567017,\n",
              " -0.054362453520298004,\n",
              " -0.19130302965641022,\n",
              " -0.15511615574359894,\n",
              " 0.4509924650192261,\n",
              " -0.056046925485134125,\n",
              " 0.6325150728225708,\n",
              " 0.13101376593112946,\n",
              " 0.4033447206020355,\n",
              " -0.2099778801202774,\n",
              " 0.7262519001960754,\n",
              " -0.07032998651266098,\n",
              " 0.7645426988601685,\n",
              " 0.9938344359397888,\n",
              " 0.02033957652747631,\n",
              " 0.17447638511657715,\n",
              " -0.4468141496181488,\n",
              " -1.3276547193527222,\n",
              " -0.2648731768131256,\n",
              " -0.10976336151361465,\n",
              " 0.7415199279785156,\n",
              " -0.45961979031562805,\n",
              " 0.1897536814212799,\n",
              " 0.0638318881392479,\n",
              " 0.06149629130959511,\n",
              " -0.09967007488012314,\n",
              " 0.7336502075195312,\n",
              " -0.06413761526346207,\n",
              " -0.1891217827796936,\n",
              " -0.24445687234401703,\n",
              " -0.47432395815849304,\n",
              " -0.14405199885368347,\n",
              " -0.6774606704711914,\n",
              " -0.7467966675758362,\n",
              " -0.861438512802124,\n",
              " 0.5196214318275452,\n",
              " 0.14422382414340973,\n",
              " -0.15533946454524994,\n",
              " -0.06327539682388306,\n",
              " -0.4872450530529022,\n",
              " 0.5644462704658508,\n",
              " 1.032560110092163,\n",
              " 0.1374681442975998,\n",
              " 0.2468148171901703,\n",
              " 0.4823891520500183,\n",
              " -0.16289877891540527,\n",
              " -0.8474594950675964,\n",
              " 0.22677911818027496,\n",
              " 0.5296151041984558,\n",
              " 0.12557700276374817,\n",
              " 0.6889568567276001,\n",
              " 0.4285523295402527,\n",
              " 0.7264259457588196,\n",
              " 0.052278000861406326,\n",
              " 1.2548617124557495,\n",
              " 0.4824894666671753,\n",
              " -0.5780812501907349,\n",
              " 0.44263482093811035,\n",
              " -0.11004526168107986,\n",
              " 0.5798296928405762,\n",
              " 0.6913082003593445,\n",
              " 0.12917852401733398,\n",
              " 0.45097988843917847,\n",
              " 0.1443747580051422,\n",
              " -0.48323410749435425,\n",
              " 0.262681782245636,\n",
              " -0.5373804569244385,\n",
              " 0.2612181007862091,\n",
              " 1.5332109928131104,\n",
              " -0.33497849106788635,\n",
              " -0.8584234714508057,\n",
              " -0.12099424004554749,\n",
              " 0.46975940465927124,\n",
              " -0.41704556345939636,\n",
              " -0.3023562729358673,\n",
              " 0.006729298736900091,\n",
              " 0.3722342848777771,\n",
              " 0.20629166066646576,\n",
              " 0.9229329824447632,\n",
              " 0.8477144837379456,\n",
              " -0.1284029334783554,\n",
              " 0.38408347964286804,\n",
              " 0.21458560228347778,\n",
              " 0.38401395082473755,\n",
              " -0.15797626972198486,\n",
              " -0.6881616115570068,\n",
              " -0.05344204604625702,\n",
              " -0.15576277673244476,\n",
              " -0.27485764026641846,\n",
              " -4.7138190269470215,\n",
              " 1.1472502946853638,\n",
              " -0.19710806012153625,\n",
              " 0.21146346628665924,\n",
              " 0.39871957898139954,\n",
              " -0.0359017550945282,\n",
              " -0.2910023331642151,\n",
              " -0.17233806848526,\n",
              " -0.8590866327285767,\n",
              " -0.29207897186279297,\n",
              " -0.5198802351951599,\n",
              " -0.5867406129837036,\n",
              " 0.6297305226325989,\n",
              " 0.305519700050354,\n",
              " 0.1401572823524475,\n",
              " -0.4748603105545044,\n",
              " -0.5593133568763733,\n",
              " 0.2906792461872101,\n",
              " -0.42561012506484985,\n",
              " 0.1727612465620041,\n",
              " -0.20777764916419983,\n",
              " -0.891983687877655,\n",
              " -0.2907566726207733,\n",
              " -0.12844185531139374,\n",
              " 0.22982512414455414,\n",
              " 0.7961352467536926,\n",
              " -0.5275991559028625,\n",
              " 0.48746681213378906,\n",
              " 0.0015168666141107678,\n",
              " 0.10057108849287033,\n",
              " -0.19485290348529816,\n",
              " 0.02856256812810898,\n",
              " 0.17053651809692383,\n",
              " -0.05285726115107536,\n",
              " 0.0569426491856575,\n",
              " -0.27377134561538696,\n",
              " -0.5230890512466431,\n",
              " -0.16032929718494415,\n",
              " -0.29630082845687866,\n",
              " -0.27530553936958313,\n",
              " 0.11373618245124817,\n",
              " 0.9082897901535034,\n",
              " 0.24302715063095093,\n",
              " 0.6416208148002625,\n",
              " 0.5646347999572754,\n",
              " -0.25327402353286743,\n",
              " -0.6521884799003601,\n",
              " -0.1521545946598053,\n",
              " -0.06550370901823044,\n",
              " 0.5766035914421082,\n",
              " 0.4209102392196655,\n",
              " -0.06473656743764877,\n",
              " 0.05876297503709793,\n",
              " -0.45727553963661194,\n",
              " 0.06108767166733742,\n",
              " 0.8958364725112915,\n",
              " -0.36628636717796326,\n",
              " 0.14007686078548431,\n",
              " 0.577181875705719,\n",
              " 0.14006316661834717,\n",
              " -0.5008761882781982,\n",
              " -0.18387305736541748,\n",
              " 0.4621230363845825,\n",
              " -0.3555099070072174,\n",
              " -0.42098698019981384,\n",
              " 0.028004685416817665,\n",
              " -1.13774836063385,\n",
              " -0.441902220249176,\n",
              " 0.0426567867398262,\n",
              " -0.07604004442691803,\n",
              " -0.20999887585639954,\n",
              " 0.1861838549375534,\n",
              " -0.09547577053308487,\n",
              " -1.3291467428207397,\n",
              " -0.706694483757019,\n",
              " -0.8307603001594543,\n",
              " 0.813284695148468,\n",
              " -0.07158366590738297,\n",
              " -0.09794814139604568,\n",
              " 0.24557414650917053,\n",
              " -0.12810871005058289,\n",
              " -0.06548894196748734,\n",
              " 0.1978825181722641,\n",
              " -0.13173913955688477,\n",
              " 0.07436143606901169,\n",
              " -0.007578194607049227,\n",
              " -0.10148505121469498,\n",
              " -0.06617563217878342,\n",
              " -1.1840711832046509,\n",
              " -0.3190862536430359,\n",
              " 0.6173363924026489,\n",
              " 1.2081345319747925,\n",
              " 0.24463960528373718,\n",
              " -0.6548506617546082,\n",
              " -0.40628817677497864,\n",
              " 0.13262231647968292,\n",
              " -0.5336574912071228,\n",
              " -0.39227309823036194,\n",
              " -0.24157367646694183,\n",
              " -0.07457894831895828,\n",
              " 0.3337547481060028,\n",
              " 0.2539590299129486,\n",
              " 0.7726778984069824,\n",
              " -0.23592643439769745,\n",
              " -0.4134071469306946,\n",
              " 0.08823780715465546,\n",
              " -0.14087975025177002,\n",
              " -0.2525143325328827,\n",
              " -1.2587977647781372,\n",
              " 1.0611134767532349,\n",
              " 0.6497206687927246,\n",
              " -0.775536298751831,\n",
              " 0.9615662097930908,\n",
              " -0.6649736166000366,\n",
              " -0.27312833070755005,\n",
              " 0.35859930515289307,\n",
              " 0.1480875164270401,\n",
              " -0.4409973919391632,\n",
              " 0.30519962310791016,\n",
              " -0.3179708421230316,\n",
              " -0.30346181988716125,\n",
              " -0.386663556098938,\n",
              " -0.8119008541107178,\n",
              " -0.9003256559371948,\n",
              " -0.15210579335689545,\n",
              " -0.2330915480852127,\n",
              " -0.1044941395521164,\n",
              " -0.378468781709671,\n",
              " -0.4813416600227356,\n",
              " -0.08389631658792496,\n",
              " 0.3126143217086792,\n",
              " -0.20463210344314575,\n",
              " -0.27104493975639343,\n",
              " 0.15639257431030273,\n",
              " 0.7142956256866455,\n",
              " 0.09514512866735458,\n",
              " 0.24290503561496735,\n",
              " 0.1158050000667572,\n",
              " 0.38275912404060364,\n",
              " 0.011625804007053375,\n",
              " 0.39521437883377075,\n",
              " 0.7633675336837769,\n",
              " -0.11670338362455368,\n",
              " -0.06850308179855347,\n",
              " -0.19494585692882538,\n",
              " -0.014844810590147972,\n",
              " -0.5725080966949463,\n",
              " 0.17233353853225708,\n",
              " 0.12623582780361176,\n",
              " -0.22925788164138794,\n",
              " -0.18757054209709167,\n",
              " -0.07410996407270432,\n",
              " 0.304327130317688,\n",
              " -0.0402066707611084,\n",
              " 0.4221647083759308,\n",
              " 0.5043947100639343,\n",
              " 1.0071957111358643,\n",
              " -0.5886744260787964,\n",
              " -1.1333577632904053,\n",
              " -0.5394506454467773,\n",
              " 0.31415751576423645,\n",
              " -0.5191509127616882,\n",
              " 1.2066231966018677,\n",
              " 0.40418216586112976,\n",
              " -1.21492600440979,\n",
              " 0.6267555952072144,\n",
              " -1.0506728887557983,\n",
              " -0.18643181025981903,\n",
              " -0.32334092259407043,\n",
              " 0.9154725074768066,\n",
              " -0.08705080300569534,\n",
              " -0.44664081931114197,\n",
              " -0.008301863446831703,\n",
              " 0.7900857329368591,\n",
              " 0.16304919123649597,\n",
              " 0.18622243404388428,\n",
              " -0.2365952730178833,\n",
              " -0.6985001564025879,\n",
              " -0.001590969506651163,\n",
              " -0.17388904094696045,\n",
              " -0.09208416193723679,\n",
              " 0.7112207412719727,\n",
              " -0.07118416577577591,\n",
              " -0.3705156445503235,\n",
              " 0.7322559356689453,\n",
              " 0.8640187382698059,\n",
              " 0.08718504756689072,\n",
              " 0.3734130561351776,\n",
              " -0.07659972459077835,\n",
              " -0.3103960156440735,\n",
              " -0.48530325293540955,\n",
              " 0.5871528387069702,\n",
              " -0.07122574001550674,\n",
              " -0.3300064504146576,\n",
              " -0.8168904185295105,\n",
              " -1.267369270324707,\n",
              " 0.21823756396770477,\n",
              " 0.8372991681098938,\n",
              " -0.34404438734054565,\n",
              " 0.506047785282135,\n",
              " -0.40105727314949036,\n",
              " 0.44113385677337646,\n",
              " -0.04780484363436699,\n",
              " -0.3000093102455139,\n",
              " -0.3327059745788574,\n",
              " -0.06588700413703918,\n",
              " 0.5178906917572021,\n",
              " -0.12201234698295593,\n",
              " -0.4831218421459198,\n",
              " 0.5817102193832397,\n",
              " -0.677533745765686,\n",
              " -0.5163298845291138,\n",
              " -0.8505939245223999,\n",
              " -0.415616899728775,\n",
              " 0.5328510403633118,\n",
              " -0.024158023297786713,\n",
              " 0.16554933786392212,\n",
              " -0.2434290200471878,\n",
              " -0.16339756548404694,\n",
              " -0.05579959601163864,\n",
              " 0.587302565574646,\n",
              " -0.1521962285041809,\n",
              " -0.3015564978122711,\n",
              " -0.1310032606124878,\n",
              " -0.7162836790084839,\n",
              " 0.03477039560675621,\n",
              " 0.5628183484077454,\n",
              " 0.6787298917770386,\n",
              " -0.11239756643772125,\n",
              " 0.6112292408943176,\n",
              " -0.5384969115257263,\n",
              " -0.6988146901130676,\n",
              " 0.1996585577726364,\n",
              " -0.1166294515132904,\n",
              " 0.3823789358139038,\n",
              " -0.12600558996200562,\n",
              " -0.40558451414108276,\n",
              " -1.673316240310669,\n",
              " 0.17436443269252777,\n",
              " -0.20716780424118042,\n",
              " -0.42602336406707764,\n",
              " -0.27807143330574036,\n",
              " 0.1268894225358963,\n",
              " -0.2489089071750641,\n",
              " -0.3094348609447479,\n",
              " 0.3368987441062927,\n",
              " 0.2447870522737503,\n",
              " 0.6193271279335022,\n",
              " -0.10677080601453781,\n",
              " 0.8617886900901794,\n",
              " 0.695437490940094,\n",
              " -0.22224445641040802,\n",
              " 0.23771314322948456,\n",
              " -0.14912015199661255,\n",
              " -0.7788679599761963,\n",
              " 0.7271231412887573,\n",
              " 0.2481394112110138,\n",
              " -0.15596002340316772,\n",
              " -0.056470271199941635,\n",
              " 0.9509419798851013,\n",
              " 0.4458785355091095,\n",
              " -0.43869224190711975,\n",
              " -0.5324495434761047,\n",
              " 0.17995904386043549,\n",
              " 0.2273031622171402,\n",
              " 0.351583331823349,\n",
              " -0.17859810590744019,\n",
              " -0.07232295721769333,\n",
              " 0.3719618320465088,\n",
              " 0.032527655363082886,\n",
              " 0.7535582184791565,\n",
              " -1.1114518642425537,\n",
              " 0.6748034954071045,\n",
              " -0.45491692423820496,\n",
              " 0.2891286313533783,\n",
              " 0.4296434819698334,\n",
              " -0.7953675985336304,\n",
              " -0.7722455263137817,\n",
              " -0.13839304447174072,\n",
              " -0.6837830543518066,\n",
              " -0.2842646837234497,\n",
              " 0.5935693979263306,\n",
              " 0.05722421035170555,\n",
              " 0.30562958121299744,\n",
              " -0.5616167187690735,\n",
              " -0.01245768740773201,\n",
              " -0.6375252604484558,\n",
              " 0.0212950948625803,\n",
              " -0.7530996203422546,\n",
              " -0.2635849714279175,\n",
              " 0.3037218749523163,\n",
              " 0.5297483205795288,\n",
              " 0.35371431708335876,\n",
              " -0.0007970120059326291,\n",
              " -0.7254049181938171,\n",
              " 0.6826987862586975,\n",
              " 0.48837336897850037,\n",
              " 0.2993488609790802,\n",
              " -0.1622321903705597,\n",
              " 0.35202497243881226,\n",
              " 0.06913198530673981,\n",
              " 0.09527890384197235,\n",
              " 0.5691472887992859,\n",
              " -0.07212528586387634,\n",
              " 0.32986709475517273,\n",
              " 0.42591169476509094,\n",
              " -0.05814375728368759,\n",
              " -0.17143839597702026,\n",
              " -0.2911953032016754,\n",
              " -0.27910423278808594,\n",
              " -0.35036420822143555,\n",
              " -0.7579910755157471,\n",
              " 0.7991662621498108,\n",
              " 0.696576714515686,\n",
              " -1.0417869091033936,\n",
              " -0.1664784848690033,\n",
              " -0.2974942624568939,\n",
              " -0.32885390520095825,\n",
              " -0.9601884484291077,\n",
              " 0.3063417077064514,\n",
              " -0.3888414800167084,\n",
              " 0.6851862072944641,\n",
              " 0.28863611817359924,\n",
              " 0.25603893399238586,\n",
              " -0.020979171618819237,\n",
              " 0.18187777698040009,\n",
              " -0.26445284485816956,\n",
              " -0.26006659865379333,\n",
              " 0.07117189466953278,\n",
              " 0.3774837851524353,\n",
              " -0.34883975982666016,\n",
              " 1.03910493850708,\n",
              " -0.2111617922782898,\n",
              " -0.047962091863155365,\n",
              " 0.1769087314605713,\n",
              " 0.07101418077945709,\n",
              " -0.1716872900724411,\n",
              " -0.35795095562934875,\n",
              " 0.035832107067108154,\n",
              " -0.37248045206069946,\n",
              " -0.23937898874282837,\n",
              " -0.32870492339134216,\n",
              " -0.44711193442344666,\n",
              " 1.3990776538848877,\n",
              " -0.2053092122077942,\n",
              " -0.0026570947375148535,\n",
              " -0.15521860122680664,\n",
              " -0.455158531665802,\n",
              " 1.0217094421386719,\n",
              " 0.6216564178466797,\n",
              " -0.02036772482097149,\n",
              " -0.20303137600421906,\n",
              " -0.6808188557624817,\n",
              " -0.8990669250488281,\n",
              " 0.3390315771102905,\n",
              " 0.19623315334320068,\n",
              " 0.11986949294805527,\n",
              " 0.27159613370895386,\n",
              " -0.23358164727687836,\n",
              " 0.5599480271339417,\n",
              " -0.03280773386359215,\n",
              " 0.5225099325180054,\n",
              " 0.14646805822849274,\n",
              " -0.6493239402770996,\n",
              " -0.18646183609962463,\n",
              " 0.3930899202823639,\n",
              " -0.10762464255094528,\n",
              " -0.34765201807022095,\n",
              " 0.22150973975658417,\n",
              " 0.39177846908569336,\n",
              " -0.3285598158836365,\n",
              " -0.08709894865751266,\n",
              " 0.7550487518310547,\n",
              " 0.4633721113204956,\n",
              " 0.11735256016254425,\n",
              " -0.2671813368797302,\n",
              " -0.7986522316932678,\n",
              " 0.253805935382843,\n",
              " 0.5352625250816345,\n",
              " -0.38075530529022217,\n",
              " 0.2531590461730957,\n",
              " -0.4009472131729126,\n",
              " -0.595918595790863,\n",
              " -0.5231034755706787,\n",
              " -0.17871297895908356,\n",
              " -0.2358800768852234,\n",
              " 0.7695401906967163,\n",
              " -0.4261331558227539,\n",
              " -0.24519596993923187,\n",
              " -0.17721153795719147,\n",
              " 0.5465059876441956,\n",
              " -0.1219465583562851,\n",
              " -0.14527784287929535,\n",
              " 0.28052663803100586,\n",
              " 0.2531508207321167,\n",
              " -0.777701735496521,\n",
              " -0.10963913798332214,\n",
              " 0.12396340817213058,\n",
              " 0.49316245317459106,\n",
              " -0.1473105102777481,\n",
              " 0.45992955565452576,\n",
              " 0.05691026523709297,\n",
              " 0.02377658151090145,\n",
              " -0.04355780407786369,\n",
              " 0.481990784406662,\n",
              " 0.17863714694976807,\n",
              " 0.6581028699874878,\n",
              " -0.18780489265918732,\n",
              " -0.4666840136051178,\n",
              " 0.1186894103884697,\n",
              " -0.007880287244915962,\n",
              " -0.009187684394419193,\n",
              " -0.4598256051540375,\n",
              " 0.918662428855896,\n",
              " -0.5285695791244507,\n",
              " -0.9275496602058411,\n",
              " 0.03892984986305237,\n",
              " -0.08570718765258789,\n",
              " 0.0033996650017797947,\n",
              " 0.2208450585603714,\n",
              " -0.2444385439157486,\n",
              " 0.15943770110607147,\n",
              " 0.3823903501033783,\n",
              " 0.4675595760345459,\n",
              " -0.6928361058235168,\n",
              " -0.19188463687896729,\n",
              " -0.5813184976577759,\n",
              " -0.32433605194091797,\n",
              " 0.2817934453487396,\n",
              " 0.6133005023002625,\n",
              " -1.1182153224945068,\n",
              " 0.10771692544221878,\n",
              " 0.9229233860969543,\n",
              " 0.7619935274124146,\n",
              " -0.3032039701938629,\n",
              " -0.24838566780090332,\n",
              " 0.41018038988113403,\n",
              " -0.18323729932308197,\n",
              " -0.3770483136177063,\n",
              " 0.06304383277893066,\n",
              " 0.41856250166893005,\n",
              " -0.3283161520957947,\n",
              " -0.42119327187538147,\n",
              " 0.40439513325691223,\n",
              " 0.5936760306358337,\n",
              " -0.4278385639190674,\n",
              " -0.2759457230567932,\n",
              " -0.5448020100593567,\n",
              " -0.34559497237205505,\n",
              " -0.019203322008252144,\n",
              " -0.24057364463806152,\n",
              " -0.47151950001716614,\n",
              " -0.24612969160079956,\n",
              " -0.3914054036140442,\n",
              " -0.4090040326118469,\n",
              " -0.12923091650009155,\n",
              " -0.4485046863555908,\n",
              " -0.39662888646125793,\n",
              " -0.2992093861103058,\n",
              " -0.2091265320777893]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Original Bert\n",
        "distances_df[distances_df.text1 == 'The bank vault was robust.']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "JcYgRD59FDQB",
        "outputId": "ad652c9d-8ec6-4960-d509-6d59d1175b9b"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         text1                               text2  distance\n",
              "12  The bank vault was robust.                                bank  0.494099\n",
              "13  The bank vault was robust.         The river bank was flooded.  0.523325\n",
              "14  The bank vault was robust.          The bank vault was robust.  1.000000\n",
              "15  The bank vault was robust.  He had to bank on her for support.  0.416074\n",
              "16  The bank vault was robust.          The bank was out of money.  0.759213\n",
              "17  The bank vault was robust.          The bank teller was a man.  0.867661"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8805308a-beb0-4cd6-9a9a-c264f7fd05da\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>distance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>bank</td>\n",
              "      <td>0.494099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The river bank was flooded.</td>\n",
              "      <td>0.523325</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>He had to bank on her for support.</td>\n",
              "      <td>0.416074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank was out of money.</td>\n",
              "      <td>0.759213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank teller was a man.</td>\n",
              "      <td>0.867661</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8805308a-beb0-4cd6-9a9a-c264f7fd05da')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8805308a-beb0-4cd6-9a9a-c264f7fd05da button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8805308a-beb0-4cd6-9a9a-c264f7fd05da');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## FinBert\n",
        "## distances_df[distances_df.text1 == 'bank']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        },
        "id": "3bSeJ7keEDmV",
        "outputId": "08ae628c-36e1-410f-d286-c5ff11077c9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  text1                               text2  distance\n",
              "0  bank                                bank  1.000000\n",
              "1  bank         The river bank was flooded.  0.207995\n",
              "2  bank          The bank vault was robust.  0.230087\n",
              "3  bank  He had to bank on her for support.  0.194959\n",
              "4  bank          The bank was out of money.  0.243639\n",
              "5  bank          The bank teller was a man.  0.343444"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-59c59b2b-f0ca-4a20-af60-8d658920fb3f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>distance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bank</td>\n",
              "      <td>bank</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>bank</td>\n",
              "      <td>The river bank was flooded.</td>\n",
              "      <td>0.207995</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bank</td>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>0.230087</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>bank</td>\n",
              "      <td>He had to bank on her for support.</td>\n",
              "      <td>0.194959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>bank</td>\n",
              "      <td>The bank was out of money.</td>\n",
              "      <td>0.243639</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>bank</td>\n",
              "      <td>The bank teller was a man.</td>\n",
              "      <td>0.343444</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-59c59b2b-f0ca-4a20-af60-8d658920fb3f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-59c59b2b-f0ca-4a20-af60-8d658920fb3f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-59c59b2b-f0ca-4a20-af60-8d658920fb3f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### “Context-based” pre-trained embeddings\n",
        "Embeddings generated for the word \"bank\" from each sentence with the word create a context-based embedding. These embeddings are the most common form of transfer learning and show the true power of the method.\n",
        "In this example, the embeddings for the word \"bank\" when it means a financial institution are far from the embeddings for it when it means a riverbank or the verb form of the word."
      ],
      "metadata": {
        "id": "31N3Y-36G8f8"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KbHOcY4_AiEq",
        "outputId": "f62f09ad-98f4-4620-f0ee-d98629d4dfb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         text1                               text2  distance\n",
              "12  The bank vault was robust.                                bank  0.494098\n",
              "13  The bank vault was robust.         The river bank was flooded.  0.523326\n",
              "14  The bank vault was robust.          The bank vault was robust.  1.000000\n",
              "15  The bank vault was robust.  He had to bank on her for support.  0.416074\n",
              "16  The bank vault was robust.          The bank was out of money.  0.759213\n",
              "17  The bank vault was robust.          The bank teller was a man.  0.867661"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-decaabf2-9eac-4051-98ee-577941f542a6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>distance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>bank</td>\n",
              "      <td>0.494098</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The river bank was flooded.</td>\n",
              "      <td>0.523326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>He had to bank on her for support.</td>\n",
              "      <td>0.416074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank was out of money.</td>\n",
              "      <td>0.759213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>The bank vault was robust.</td>\n",
              "      <td>The bank teller was a man.</td>\n",
              "      <td>0.867661</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-decaabf2-9eac-4051-98ee-577941f542a6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-decaabf2-9eac-4051-98ee-577941f542a6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-decaabf2-9eac-4051-98ee-577941f542a6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ],
      "source": [
        "distances_df[distances_df.text1 == 'The bank vault was robust.']"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### \"Context-averaged\" pre-trained embeddings\n",
        "When all the embeddings are averaged together, they create a context-averaged embedding. This style of embedding might be useful in some applications where one needs to get the average meaning of the word.\n",
        "\n",
        "Surprisingly, the context-free and context-averaged versions of the word are not the same as shown by the cosine distance of $0.65$ between them."
      ],
      "metadata": {
        "id": "vz2X86-hHAy6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eWQ_2Da7AiEr",
        "outputId": "5b8037b0-9ca8-4aaa-f71e-2459bfcaab71",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Distance between context-free and context-averaged = 0.6590345238888202\n"
          ]
        }
      ],
      "source": [
        "cos_dist = 1 - cosine(target_word_embeddings[0], np.sum(target_word_embeddings, axis=0))\n",
        "print(f'Distance between context-free and context-averaged = {cos_dist}')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "name": "BERT Word Embeddings.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "603f391aad42446980c41f1a34ce3481": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e805150389f546c7ada90adf1cfc8006",
              "IPY_MODEL_688dcd76bd434b29a17ca62eabdbc2c0",
              "IPY_MODEL_52b76b32261f4766b22c733f4a0f3abe"
            ],
            "layout": "IPY_MODEL_e90987d9d8864ac9a52b14a9076d4121"
          }
        },
        "e805150389f546c7ada90adf1cfc8006": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d09cb09150b4802bcb22668636bce1b",
            "placeholder": "​",
            "style": "IPY_MODEL_7ddd3144dfaf4c8ea5838d4767581088",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "688dcd76bd434b29a17ca62eabdbc2c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_684e1618fde84e58b9546cfceaa54bc6",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d7239dcd186d4aa7b7b893233ef75624",
            "value": 570
          }
        },
        "52b76b32261f4766b22c733f4a0f3abe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a0716c071ea402dbf375a65b1a2f2bd",
            "placeholder": "​",
            "style": "IPY_MODEL_97810075215742c69afcd77d0651d40a",
            "value": " 570/570 [00:00&lt;00:00, 9.72kB/s]"
          }
        },
        "e90987d9d8864ac9a52b14a9076d4121": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d09cb09150b4802bcb22668636bce1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ddd3144dfaf4c8ea5838d4767581088": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "684e1618fde84e58b9546cfceaa54bc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7239dcd186d4aa7b7b893233ef75624": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8a0716c071ea402dbf375a65b1a2f2bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97810075215742c69afcd77d0651d40a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8540b0a6159e4d6eb64e18aa0504233a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_28ae02c0458640f0a1491c97faaa5715",
              "IPY_MODEL_93f20db69e2b449aaef50fd108ea8578",
              "IPY_MODEL_8792c1eb666b46748fbdb0baaedcb2b1"
            ],
            "layout": "IPY_MODEL_718b9ca932af473e8ce793bea0197f10"
          }
        },
        "28ae02c0458640f0a1491c97faaa5715": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fdcfd3da5c44a289c218aa0942a1c5a",
            "placeholder": "​",
            "style": "IPY_MODEL_8871cb97f963455ab328489da803d0e9",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "93f20db69e2b449aaef50fd108ea8578": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0aad6377798f466283bba6af0c367d46",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_193855bc315b4922b232db772dec7452",
            "value": 440473133
          }
        },
        "8792c1eb666b46748fbdb0baaedcb2b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42389b2495cf44a29f833aed7114d0f0",
            "placeholder": "​",
            "style": "IPY_MODEL_98aadd6045694768a25814f629ffda89",
            "value": " 440M/440M [00:03&lt;00:00, 123MB/s]"
          }
        },
        "718b9ca932af473e8ce793bea0197f10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fdcfd3da5c44a289c218aa0942a1c5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8871cb97f963455ab328489da803d0e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0aad6377798f466283bba6af0c367d46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "193855bc315b4922b232db772dec7452": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "42389b2495cf44a29f833aed7114d0f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98aadd6045694768a25814f629ffda89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd0a9fa92d4442c0bb21cae97e5b1538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d976b5f1e2034b5ea2bda4dc877e1d3d",
              "IPY_MODEL_f3ded921ee124b02920419695181173d",
              "IPY_MODEL_dd329ca3ac8a411793c1a6b863f751ed"
            ],
            "layout": "IPY_MODEL_7aa436546c4e4bf2b270f7fec6e91590"
          }
        },
        "d976b5f1e2034b5ea2bda4dc877e1d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2defd2a76be94f84ba957b21477909c1",
            "placeholder": "​",
            "style": "IPY_MODEL_80f82c5f6cfe494fa56eef70dfcdd05c",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "f3ded921ee124b02920419695181173d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4a2612024dd45a981ed7392a7513470",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e1f8f179f6e40f7978b005419b2929e",
            "value": 231508
          }
        },
        "dd329ca3ac8a411793c1a6b863f751ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_800b1520c0ac4bc3b43074ab21e8a5f4",
            "placeholder": "​",
            "style": "IPY_MODEL_3580f7c0b9b04ac7b9e32e5c2a9989a6",
            "value": " 232k/232k [00:00&lt;00:00, 1.99MB/s]"
          }
        },
        "7aa436546c4e4bf2b270f7fec6e91590": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2defd2a76be94f84ba957b21477909c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80f82c5f6cfe494fa56eef70dfcdd05c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e4a2612024dd45a981ed7392a7513470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e1f8f179f6e40f7978b005419b2929e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "800b1520c0ac4bc3b43074ab21e8a5f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3580f7c0b9b04ac7b9e32e5c2a9989a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "65c4004f46f74db38d53e9c611e33879": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4829b7f750314cff996cb820e559dc2c",
              "IPY_MODEL_fc4b069dc76a46f9ab0ea6fc6656c2ea",
              "IPY_MODEL_8d3bdcd379ef43f181e1814d41b4d95f"
            ],
            "layout": "IPY_MODEL_384c4c1f7752480cae2af7c28c854dab"
          }
        },
        "4829b7f750314cff996cb820e559dc2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8bed55b4fea84dcf92a6401d34cb0d67",
            "placeholder": "​",
            "style": "IPY_MODEL_08999f81dc6446ae9adba574dd7ee444",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "fc4b069dc76a46f9ab0ea6fc6656c2ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5aa090da8fd4e0ab0dcd6cd6de1f512",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1aebcc0db0a1491b94f64a34fa62fc68",
            "value": 28
          }
        },
        "8d3bdcd379ef43f181e1814d41b4d95f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67fe0f157fec443693c94a40e78515c2",
            "placeholder": "​",
            "style": "IPY_MODEL_f5bdbc4307414c71956af3baa51a0261",
            "value": " 28.0/28.0 [00:00&lt;00:00, 461B/s]"
          }
        },
        "384c4c1f7752480cae2af7c28c854dab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8bed55b4fea84dcf92a6401d34cb0d67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08999f81dc6446ae9adba574dd7ee444": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5aa090da8fd4e0ab0dcd6cd6de1f512": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1aebcc0db0a1491b94f64a34fa62fc68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67fe0f157fec443693c94a40e78515c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5bdbc4307414c71956af3baa51a0261": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}