Created
February 1, 2022 00:36
-
-
Save yazdipour/13a5163951e0bea52411c21ad3a2fec5 to your computer and use it in GitHub Desktop.
bleu.ipynb
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "nbformat": 4, | |
| "nbformat_minor": 0, | |
| "metadata": { | |
| "colab": { | |
| "name": "bleu.ipynb", | |
| "provenance": [], | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3" | |
| }, | |
| "language_info": { | |
| "name": "python" | |
| }, | |
| "widgets": { | |
| "application/vnd.jupyter.widget-state+json": { | |
| "390304a873bf45bb9b6673db91c9f9ad": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HBoxModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "HBoxView", | |
| "_dom_classes": [], | |
| "_model_name": "HBoxModel", | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_view_count": null, | |
| "_view_module_version": "1.5.0", | |
| "box_style": "", | |
| "layout": "IPY_MODEL_1fb6841bfbf045d9ab8d0ee00396fd66", | |
| "_model_module": "@jupyter-widgets/controls", | |
| "children": [ | |
| "IPY_MODEL_0e11898938604b0c9d9f8041135289ed", | |
| "IPY_MODEL_dc9e50389c564a0a933c382e715d29d6", | |
| "IPY_MODEL_b2fb3e7601e34c82950aeaf5e58d32bc" | |
| ] | |
| } | |
| }, | |
| "1fb6841bfbf045d9ab8d0ee00396fd66": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "model_module_version": "1.2.0", | |
| "state": { | |
| "_view_name": "LayoutView", | |
| "grid_template_rows": null, | |
| "right": null, | |
| "justify_content": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "overflow": null, | |
| "_model_module_version": "1.2.0", | |
| "_view_count": null, | |
| "flex_flow": null, | |
| "width": null, | |
| "min_width": null, | |
| "border": null, | |
| "align_items": null, | |
| "bottom": null, | |
| "_model_module": "@jupyter-widgets/base", | |
| "top": null, | |
| "grid_column": null, | |
| "overflow_y": null, | |
| "overflow_x": null, | |
| "grid_auto_flow": null, | |
| "grid_area": null, | |
| "grid_template_columns": null, | |
| "flex": null, | |
| "_model_name": "LayoutModel", | |
| "justify_items": null, | |
| "grid_row": null, | |
| "max_height": null, | |
| "align_content": null, | |
| "visibility": null, | |
| "align_self": null, | |
| "height": null, | |
| "min_height": null, | |
| "padding": null, | |
| "grid_auto_rows": null, | |
| "grid_gap": null, | |
| "max_width": null, | |
| "order": null, | |
| "_view_module_version": "1.2.0", | |
| "grid_template_areas": null, | |
| "object_position": null, | |
| "object_fit": null, | |
| "grid_auto_columns": null, | |
| "margin": null, | |
| "display": null, | |
| "left": null | |
| } | |
| }, | |
| "0e11898938604b0c9d9f8041135289ed": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HTMLModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "HTMLView", | |
| "style": "IPY_MODEL_2f943945c98543708c5b810002ea239e", | |
| "_dom_classes": [], | |
| "description": "", | |
| "_model_name": "HTMLModel", | |
| "placeholder": "", | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "value": "Downloading: 100%", | |
| "_view_count": null, | |
| "_view_module_version": "1.5.0", | |
| "description_tooltip": null, | |
| "_model_module": "@jupyter-widgets/controls", | |
| "layout": "IPY_MODEL_1194cdd9e0c14cc4a9695a379ca383a6" | |
| } | |
| }, | |
| "dc9e50389c564a0a933c382e715d29d6": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "FloatProgressModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "ProgressView", | |
| "style": "IPY_MODEL_5be4a682d84f47779a44029cea46ed77", | |
| "_dom_classes": [], | |
| "description": "", | |
| "_model_name": "FloatProgressModel", | |
| "bar_style": "success", | |
| "max": 482, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "value": 482, | |
| "_view_count": null, | |
| "_view_module_version": "1.5.0", | |
| "orientation": "horizontal", | |
| "min": 0, | |
| "description_tooltip": null, | |
| "_model_module": "@jupyter-widgets/controls", | |
| "layout": "IPY_MODEL_89b3f677e32c4a1db278b2fedea964dc" | |
| } | |
| }, | |
| "b2fb3e7601e34c82950aeaf5e58d32bc": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HTMLModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "HTMLView", | |
| "style": "IPY_MODEL_371c5337fc3d4278935730be683e76a6", | |
| "_dom_classes": [], | |
| "description": "", | |
| "_model_name": "HTMLModel", | |
| "placeholder": "", | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "value": " 482/482 [00:00<00:00, 12.7kB/s]", | |
| "_view_count": null, | |
| "_view_module_version": "1.5.0", | |
| "description_tooltip": null, | |
| "_model_module": "@jupyter-widgets/controls", | |
| "layout": "IPY_MODEL_0e7b49c8eaaa4fe8ab88270a7b81c1dc" | |
| } | |
| }, | |
| "2f943945c98543708c5b810002ea239e": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "DescriptionStyleModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "StyleView", | |
| "_model_name": "DescriptionStyleModel", | |
| "description_width": "", | |
| "_view_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.5.0", | |
| "_view_count": null, | |
| "_view_module_version": "1.2.0", | |
| "_model_module": "@jupyter-widgets/controls" | |
| } | |
| }, | |
| "1194cdd9e0c14cc4a9695a379ca383a6": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "model_module_version": "1.2.0", | |
| "state": { | |
| "_view_name": "LayoutView", | |
| "grid_template_rows": null, | |
| "right": null, | |
| "justify_content": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "overflow": null, | |
| "_model_module_version": "1.2.0", | |
| "_view_count": null, | |
| "flex_flow": null, | |
| "width": null, | |
| "min_width": null, | |
| "border": null, | |
| "align_items": null, | |
| "bottom": null, | |
| "_model_module": "@jupyter-widgets/base", | |
| "top": null, | |
| "grid_column": null, | |
| "overflow_y": null, | |
| "overflow_x": null, | |
| "grid_auto_flow": null, | |
| "grid_area": null, | |
| "grid_template_columns": null, | |
| "flex": null, | |
| "_model_name": "LayoutModel", | |
| "justify_items": null, | |
| "grid_row": null, | |
| "max_height": null, | |
| "align_content": null, | |
| "visibility": null, | |
| "align_self": null, | |
| "height": null, | |
| "min_height": null, | |
| "padding": null, | |
| "grid_auto_rows": null, | |
| "grid_gap": null, | |
| "max_width": null, | |
| "order": null, | |
| "_view_module_version": "1.2.0", | |
| "grid_template_areas": null, | |
| "object_position": null, | |
| "object_fit": null, | |
| "grid_auto_columns": null, | |
| "margin": null, | |
| "display": null, | |
| "left": null | |
| } | |
| }, | |
| "5be4a682d84f47779a44029cea46ed77": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "ProgressStyleModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "StyleView", | |
| "_model_name": "ProgressStyleModel", | |
| "description_width": "", | |
| "_view_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.5.0", | |
| "_view_count": null, | |
| "_view_module_version": "1.2.0", | |
| "bar_color": null, | |
| "_model_module": "@jupyter-widgets/controls" | |
| } | |
| }, | |
| "89b3f677e32c4a1db278b2fedea964dc": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "model_module_version": "1.2.0", | |
| "state": { | |
| "_view_name": "LayoutView", | |
| "grid_template_rows": null, | |
| "right": null, | |
| "justify_content": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "overflow": null, | |
| "_model_module_version": "1.2.0", | |
| "_view_count": null, | |
| "flex_flow": null, | |
| "width": null, | |
| "min_width": null, | |
| "border": null, | |
| "align_items": null, | |
| "bottom": null, | |
| "_model_module": "@jupyter-widgets/base", | |
| "top": null, | |
| "grid_column": null, | |
| "overflow_y": null, | |
| "overflow_x": null, | |
| "grid_auto_flow": null, | |
| "grid_area": null, | |
| "grid_template_columns": null, | |
| "flex": null, | |
| "_model_name": "LayoutModel", | |
| "justify_items": null, | |
| "grid_row": null, | |
| "max_height": null, | |
| "align_content": null, | |
| "visibility": null, | |
| "align_self": null, | |
| "height": null, | |
| "min_height": null, | |
| "padding": null, | |
| "grid_auto_rows": null, | |
| "grid_gap": null, | |
| "max_width": null, | |
| "order": null, | |
| "_view_module_version": "1.2.0", | |
| "grid_template_areas": null, | |
| "object_position": null, | |
| "object_fit": null, | |
| "grid_auto_columns": null, | |
| "margin": null, | |
| "display": null, | |
| "left": null | |
| } | |
| }, | |
| "371c5337fc3d4278935730be683e76a6": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "DescriptionStyleModel", | |
| "model_module_version": "1.5.0", | |
| "state": { | |
| "_view_name": "StyleView", | |
| "_model_name": "DescriptionStyleModel", | |
| "description_width": "", | |
| "_view_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.5.0", | |
| "_view_count": null, | |
| "_view_module_version": "1.2.0", | |
| "_model_module": "@jupyter-widgets/controls" | |
| } | |
| }, | |
| "0e7b49c8eaaa4fe8ab88270a7b81c1dc": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "model_module_version": "1.2.0", | |
| "state": { | |
| "_view_name": "LayoutView", | |
| "grid_template_rows": null, | |
| "right": null, | |
| "justify_content": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "overflow": null, | |
| "_model_module_version": "1.2.0", | |
| "_view_count": null, | |
| "flex_flow": null, | |
| "width": null, | |
| "min_width": null, | |
| "border": null, | |
| "align_items": null, | |
| "bottom": null, | |
| "_model_module": "@jupyter-widgets/base", | |
| "top": null, | |
| "grid_column": null, | |
| "overflow_y": null, | |
| "overflow_x": null, | |
| "grid_auto_flow": null, | |
| "grid_area": null, | |
| "grid_template_columns": null, | |
| "flex": null, | |
| "_model_name": "LayoutModel", | |
| "justify_items": null, | |
| "grid_row": null, | |
| "max_height": null, | |
| "align_content": null, | |
| "visibility": null, | |
| "align_self": null, | |
| "height": null, | |
| "min_height": null, | |
| "padding": null, | |
| "grid_auto_rows": null, | |
| "grid_gap": null, | |
| "max_width": null, | |
| "order": null, | |
| "_view_module_version": "1.2.0", | |
| "grid_template_areas": null, | |
| "object_position": null, | |
| "object_fit": null, | |
| "grid_auto_columns": null, | |
| "margin": null, | |
| "display": null, | |
| "left": null | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/yazdipour/13a5163951e0bea52411c21ad3a2fec5/bleu.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "U8ZiflDQijqf", | |
| "outputId": "d35e284f-bc5d-4f3c-a777-9f0c2798c61b" | |
| }, | |
| "source": [ | |
| "! pip install datasets transformers sacrebleu -qqq" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "\u001b[K |████████████████████████████████| 270 kB 12.1 MB/s \n", | |
| "\u001b[K |████████████████████████████████| 125 kB 56.2 MB/s \n", | |
| "\u001b[K |████████████████████████████████| 1.3 MB 53.2 MB/s \n", | |
| "\u001b[K |████████████████████████████████| 243 kB 77.6 MB/s \n", | |
| "\u001b[K |████████████████████████████████| 160 kB 51.8 MB/s \n", | |
| "\u001b[K |████████████████████████████████| 271 kB 76.0 MB/s \n", | |
| "\u001b[?25h" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 49, | |
| "referenced_widgets": [ | |
| "d57c8498de714ffba31f94f7857584f2" | |
| ] | |
| }, | |
| "id": "u0JrLKVKirdc", | |
| "outputId": "c0842e9f-bad0-428b-8d8a-1c26149f5d73" | |
| }, | |
| "source": [ | |
| "from datasets import load_metric\n", | |
| "metric = load_metric(\"sacrebleu\")" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "d57c8498de714ffba31f94f7857584f2", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/2.37k [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "dCnU6SoFjBES", | |
| "outputId": "a535c576-8467-44ea-9e50-d8bcc12e52ba" | |
| }, | |
| "source": [ | |
| "metric" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "Metric(name: \"sacrebleu\", features: {'predictions': Value(dtype='string', id='sequence'), 'references': Sequence(feature=Value(dtype='string', id='sequence'), length=-1, id='references')}, usage: \"\"\"\n", | |
| "Produces BLEU scores along with its sufficient statistics\n", | |
| "from a source against one or more references.\n", | |
| "\n", | |
| "Args:\n", | |
| " predictions: The system stream (a sequence of segments).\n", | |
| " references: A list of one or more reference streams (each a sequence of segments).\n", | |
| " smooth_method: The smoothing method to use. (Default: 'exp').\n", | |
| " smooth_value: The smoothing value. Only valid for 'floor' and 'add-k'. (Defaults: floor: 0.1, add-k: 1).\n", | |
| " tokenize: Tokenization method to use for BLEU. If not provided, defaults to 'zh' for Chinese, 'ja-mecab' for\n", | |
| " Japanese and '13a' (mteval) otherwise.\n", | |
| " lowercase: Lowercase the data. If True, enables case-insensitivity. (Default: False).\n", | |
| " force: Insist that your tokenized input is actually detokenized.\n", | |
| "\n", | |
| "Returns:\n", | |
| " 'score': BLEU score,\n", | |
| " 'counts': Counts,\n", | |
| " 'totals': Totals,\n", | |
| " 'precisions': Precisions,\n", | |
| " 'bp': Brevity penalty,\n", | |
| " 'sys_len': predictions length,\n", | |
| " 'ref_len': reference length,\n", | |
| "\n", | |
| "Examples:\n", | |
| "\n", | |
| " >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]\n", | |
| " >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]\n", | |
| " >>> sacrebleu = datasets.load_metric(\"sacrebleu\")\n", | |
| " >>> results = sacrebleu.compute(predictions=predictions, references=references)\n", | |
| " >>> print(list(results.keys()))\n", | |
| " ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']\n", | |
| " >>> print(round(results[\"score\"], 1))\n", | |
| " 100.0\n", | |
| "\"\"\", stored examples: 0)" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 7 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "8Z0bqRHRjIhr", | |
| "outputId": "f6629eea-c4b0-4538-ff5f-02515da45466" | |
| }, | |
| "source": [ | |
| "\n", | |
| "fake_preds = [\"hello there\", \"general kenobi\"]\n", | |
| "fake_labels = [[\"hello there\"], [\"general kenobi\"]]\n", | |
| "metric.compute(predictions=fake_preds, references=fake_labels)" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "data": { | |
| "text/plain": [ | |
| "{'bp': 1.0,\n", | |
| " 'counts': [4, 2, 0, 0],\n", | |
| " 'precisions': [100.0, 100.0, 0.0, 0.0],\n", | |
| " 'ref_len': 4,\n", | |
| " 'score': 0.0,\n", | |
| " 'sys_len': 4,\n", | |
| " 'totals': [4, 2, 0, 0]}" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "execution_count": 8 | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "h1bvxTJMj6rF" | |
| }, | |
| "source": [ | |
| "!pip install bert-score -qqq" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "_S_ZsMTOj5vs" | |
| }, | |
| "source": [ | |
| "from bert_score import BERTScorer" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 247, | |
| "referenced_widgets": [ | |
| "390304a873bf45bb9b6673db91c9f9ad", | |
| "1fb6841bfbf045d9ab8d0ee00396fd66", | |
| "0e11898938604b0c9d9f8041135289ed", | |
| "dc9e50389c564a0a933c382e715d29d6", | |
| "b2fb3e7601e34c82950aeaf5e58d32bc", | |
| "2f943945c98543708c5b810002ea239e", | |
| "1194cdd9e0c14cc4a9695a379ca383a6", | |
| "5be4a682d84f47779a44029cea46ed77", | |
| "89b3f677e32c4a1db278b2fedea964dc", | |
| "371c5337fc3d4278935730be683e76a6", | |
| "0e7b49c8eaaa4fe8ab88270a7b81c1dc", | |
| "b379ff12ac094b5dbd1621ec55170bc5", | |
| "7bfab52284804128ad35d0849ef5cefc", | |
| "2ed5f2524710496796859b7d554d26d6", | |
| "052ac94dbb974784a500c30c30b0f397" | |
| ] | |
| }, | |
| "id": "jgSEPd0vkDMX", | |
| "outputId": "a84ac9fa-770f-4c29-ec8f-e4a743d98f46" | |
| }, | |
| "source": [ | |
| "refs = [['The dog bit the guy.', 'The dog had bit the man.'], ['It was not unexpected.', 'No one was surprised.'], ['The man bit him first.', 'The man had bitten the dog.']]\n", | |
| "hyps = ['The dog bit the man.', \"It wasn't surprising.\", 'The man had just bitten him.']\n", | |
| "scorer = BERTScorer(lang=\"en\", rescale_with_baseline=True)" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "390304a873bf45bb9b6673db91c9f9ad", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/482 [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "b379ff12ac094b5dbd1621ec55170bc5", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/878k [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "7bfab52284804128ad35d0849ef5cefc", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/446k [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "2ed5f2524710496796859b7d554d26d6", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/1.29M [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "052ac94dbb974784a500c30c30b0f397", | |
| "version_minor": 0, | |
| "version_major": 2 | |
| }, | |
| "text/plain": [ | |
| "Downloading: 0%| | 0.00/1.33G [00:00<?, ?B/s]" | |
| ] | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "stream", | |
| "name": "stderr", | |
| "text": [ | |
| "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaModel: ['lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.weight']\n", | |
| "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", | |
| "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" | |
| ] | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "KbBvDBDQkQz1" | |
| }, | |
| "source": [ | |
| "P, R, F1 = scorer.score(hyps, refs)" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/" | |
| }, | |
| "id": "abzXU2UYkZup", | |
| "outputId": "7e1edcc6-ff28-449c-f8a8-69c30abf93d7" | |
| }, | |
| "source": [ | |
| "print(F1.mean())\n", | |
| "print(R.mean())\n", | |
| "print(P.mean())" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": [ | |
| "tensor(0.7998)\n", | |
| "tensor(0.8056)\n", | |
| "tensor(0.8044)\n" | |
| ] | |
| } | |
| ] | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment