Skip to content

Instantly share code, notes, and snippets.

@alonsosilvaallende

alonsosilvaallende/gpt-encoder-decoder.ipynb

Created October 24, 2023 06:06
Show Gist options
  • Save alonsosilvaallende/218fc71775ea1056e9c0233737dad558 to your computer and use it in GitHub Desktop.
Save alonsosilvaallende/218fc71775ea1056e9c0233737dad558 to your computer and use it in GitHub Desktop.
Download ZIP
GPT-encoder-decoder.ipynb
Display the source blob
Display the rendered blob
Raw
gpt-encoder-decoder.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyMKFa76Xi3pkCE2MoeuiBM1",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"e31fdcbcd4284271a3fc89db929df133": {
"model_module": "jupyter-vue",
"model_name": "HtmlModel",
"model_module_version": "^1.10.1",
"state": {
"_dom_classes": [],
"_events": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vue",
"_model_module_version": "^1.10.1",
"_model_name": "HtmlModel",
"_view_count": null,
"_view_module": "jupyter-vue",
"_view_module_version": "^1.10.1",
"_view_name": "VueView",
"attributes": {},
"children": [],
"class_": null,
"layout": null,
"slot": null,
"style_": "display: none",
"tag": "span",
"v_model": "!!disabled!!",
"v_on": null,
"v_slots": []
}
},
"fa81ea134de14d9c8945e658f74c8e2c": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"071ed4a34d6a480b9d6a3171c828301a": {
"model_module": "jupyter-vuetify",
"model_name": "SheetModel",
"model_module_version": "^1.8.5",
"state": {
"_dom_classes": [],
"_events": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_metadata": null,
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.5",
"_model_name": "SheetModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.5",
"_view_name": "VuetifyView",
"attributes": {},
"children": [
"IPY_MODEL_a4bf55449be24dfca9a1a95a49a8bd34",
"IPY_MODEL_51bcf572824d4f7cb5c2f044e33b0718",
"IPY_MODEL_faff82eeaf3448f8aa2450653525fdc1",
"IPY_MODEL_7ca09aa6ca4e4998846de0b4c9f6423c",
"IPY_MODEL_7f7ca25393df4f6f826b54c06a4a8e10",
"IPY_MODEL_6f3bd51f4df44c3c82d48c2994ad248c",
"IPY_MODEL_b278f27619374405a45dddc382ecc617",
"IPY_MODEL_ac3242989ff040ceaf8589e06245da56",
"IPY_MODEL_dd99642268244a80b0c15467cbbbf11c",
"IPY_MODEL_be21d9a188db45dd9e47184e6669b63d",
"IPY_MODEL_edd1e18c5c5042039453c1766288609c"
],
"class_": "d-flex ma-0",
"color": null,
"dark": null,
"elevation": 0,
"height": null,
"layout": null,
"light": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"slot": null,
"style_": "flex-direction: column; align-items: stretch; row-gap: 12px;;",
"tag": null,
"tile": null,
"v_model": "!!disabled!!",
"v_on": null,
"v_slots": [],
"width": null
}
},
"3efd7a6cfdfa44cf91b2a051043478f7": {
"model_module": "@jupyter-widgets/controls",
"model_name": "VBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "VBoxModel",
"_view_count": 1,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "VBoxView",
"box_style": "",
"children": [
"IPY_MODEL_071ed4a34d6a480b9d6a3171c828301a"
],
"layout": "IPY_MODEL_fa81ea134de14d9c8945e658f74c8e2c"
}
},
"3a8d1ff4947647e2bd6a128599f2c252": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"6cb5ff73d0df4c0f8f078ec1bcad7fa7": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"8a97a4ea57934dd895e196bad61e5792": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"36c452da204048f5ac19e32cc531d01d": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"97ba1aec3c924319b9cc0fba11138e8a": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"0986a78a60c944cea088ed86009b07af": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"8670e01752f54508ba5b158ef51cb032": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"07be2cdb751b46faa1ad70006b0e6371": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"78a5a97593ad409095ad5f6edbc4eae6": {
"model_module": "jupyter-vue",
"model_name": "TemplateModel",
"model_module_version": "^1.10.1",
"state": {
"_model_module": "jupyter-vue",
"_model_module_version": "^1.10.1",
"_model_name": "TemplateModel",
"_view_count": null,
"_view_module": null,
"_view_module_version": "",
"_view_name": null,
"template": "<template>\n <v-slide-x-transition appear>\n <div class=\"solara-data-table__viewport\">\n <v-data-table dense hide-default-header :headers=\"[...headers]\" :items=\"items\"\n :footer-props=\"{ 'items-per-page-options': [10, 20, 50, 100] }\" :options.sync=\"options\"\n :items_per_page.sync=\"items_per_page\" :server-items-length=\"total_length\" :class=\"[\n 'elevation-1',\n 'solara-data-table',\n scrollable && 'solara-data-table--scrollable',\n ]\" :style=\"scrollable && height != null && `height: ${height}`\">\n <template v-slot:header=\"props\">\n <thead>\n <tr>\n <th style=\"padding: 0 10px; width: 40px\">#</th>\n <th style=\"padding: 0 1px; width: 30px\" v-if=\"selection_enabled\">\n <v-btn icon color=\"primary\" text small @click=\"apply_filter\">\n <v-icon>filter_list</v-icon>\n </v-btn>\n </th>\n <th style=\"padding: 0 1px\" v-for=\"(header, index) in headers_selections\" :key=\"header.text\">\n <v-icon style=\"padding: 0 1px\" :key=\"index\" :color=\"selection_colors[index]\">brightness_1</v-icon>\n </th>\n <v-slide-x-transition :key=\"header.text\" v-for=\"header in headers\">\n <th class=\"text-no-wrap\">\n {{ header.text }}\n <v-menu open-on-hover bottom offset-y @input=\"isOpen => onHeaderHover({ isOpen, header })\"\n v-if=\"(column_actions && column_actions.length) || column_header_widget\">\n <template v-slot:activator=\"{ on, attrs }\">\n <v-icon v-bind=\"attrs\" v-on=\"on\" small class=\"solara-data-table-menu\">mdi-dots-vertical</v-icon>\n </template>\n <v-sheet v-if=\"header.value === column_header_hover\" class=\"solara-data-table-column-header-sheet\">\n <jupyter-widget v-if=\"column_header_widget\" :widget=\"column_header_widget\"></jupyter-widget>\n </v-sheet>\n <v-list v-if=\"column_actions && column_actions.length\">\n <v-subheader>Actions:</v-subheader>\n <v-list-item link @click=\"on_column_action([header.value, index])\"\n v-for=\"(action, index) in column_actions\" :key=\"index\">\n <v-list-item-icon><v-icon>{{ action.icon }}</v-icon></v-list-item-icon>\n <v-list-item-title>{{ action.name }}</v-list-item-title>\n </v-list-item>\n </v-list>\n </v-menu>\n </th>\n </v-slide-x-transition>\n </tr>\n </thead>\n </template>\n <template v-slot:item=\"props\">\n <!-- @click=\"on_row_clicked(props.item.__row__)\" -->\n <tr :class=\"{ highlightedRow: props.item.__row__ === highlighted }\">\n <td style=\"padding: 0 10px\" class=\"text-xs-left\">\n <i>{{ props.item.__row__ }}</i>\n </td>\n <td style=\"padding: 0 1px\" class=\"text-xs-left\" v-if=\"selection_enabled\">\n <v-checkbox hide-details style=\"margin-top: 0; padding-top: 0\"\n :input-value=\"checked.indexOf(props.item.__row__) != -1\" :key=\"props.item.__row__\"\n @change=\"(value) => select({ checked: value, row: props.item.__row__ })\" />\n </td>\n <td style=\"padding: 0 1px\" :key=\"header.text\" v-for=\"(header, index) in headers_selections\">\n <v-fade-transition leave-absolute>\n <v-icon v-if=\"props.item[header.value]\" v-model=\"props.item[header.value]\"\n :color=\"selection_colors[index]\">brightness_1</v-icon>\n </v-fade-transition>\n </td>\n <td v-for=\"header in headers\" class=\"text-truncate text-no-wrap\" :key=\"header.text\"\n :title=\"props.item[header.value]\">\n <v-slide-x-transition appear>\n <!-- <span @click=\"on_item_click([props.item.__row__, header.value])\">{{ props.item[header.value] }}</span> -->\n <span>\n {{ props.item[header.value] }}\n <v-menu open-on-hover bottom offset-y v-if=\"cell_actions.length\">\n <template v-slot:activator=\"{ on, attrs }\">\n <v-icon v-bind=\"attrs\" v-on=\"on\" small class=\"solara-data-table-menu\">mdi-dots-vertical</v-icon>\n </template>\n <v-list v-for=\"(action, index) in cell_actions\" :key=\"index\">\n <v-list-item link @click=\"on_cell_action([props.item.__row__, header.value, index])\">\n <v-list-item-icon><v-icon>{{ action.icon }}</v-icon></v-list-item-icon>\n <v-list-item-title>{{ action.name }}</v-list-item-title>\n </v-list-item>\n </v-list>\n </v-menu>\n </span>\n\n </v-slide-x-transition>\n </td>\n </tr>\n </template>\n </v-data-table>\n </div>\n </v-slide-x-transition>\n</template>\n\n<script>\nmodule.exports = {\n methods: {\n onHeaderHover({ header, isOpen }) {\n if (isOpen) {\n // if isOpen is true, we clicked, and we set it header.value\n this.column_header_hover = header.value\n } else {\n // if false, we only 'unset' if the current header equals the current open menu\n // because sometimes the menu from another column is closed after opening the new one\n if (this.column_header_hover == header.value) {\n this.column_header_hover = null\n }\n }\n }\n }\n}\n</script>\n<style id=\"solara_table\">\n.highlightedRow {\n background-color: #e3f2fd;\n}\n\n.solara-data-table table {\n table-layout: fixed;\n}\n\n.solara-data-table--scrollable .v-data-table__wrapper {\n overflow-y: auto;\n height: calc(100% - 59px);\n}\n\n.solara-data-table--scrollable thead>tr {\n position: sticky;\n top: 0;\n}\n\n.solara-data-table--scrollable .v-data-table__wrapper,\n.solara-data-table--scrollable .v-data-table__wrapper>table,\n.solara-data-table--scrollable .v-data-table__wrapper>table thead,\n.solara-data-table--scrollable .v-data-table__wrapper>table thead * {\n background-color: inherit;\n}\n\n/* prevent checkboxes overlaying the table header */\n.solara-data-table--scrollable .v-data-table__wrapper>table thead {\n position: relative;\n z-index: 1;\n}\n\n/* avoid margins to collapse, to avoid a white background */\n.solara-data-table-column-header-sheet {\n overflow: auto;\n}\n\n.solara-data-table.v-data-table th,\n.solara-data-table.v-data-table td {\n padding-left: 4px;\n padding-right: 0;\n}\n\n.v-data-table .solara-data-table-menu {\n opacity: 0;\n transition: opacity 0.5s;\n}\n\n.v-data-table th:hover .solara-data-table-menu,\n.v-data-table td:hover .solara-data-table-menu {\n opacity: 1;\n}\n\n.solara-data-table__viewport {\n overflow-x: auto;\n width: 100%;\n max-height: 100%;\n}\n\n.solara-data-table.v-data-table,\n.solara-data-table.v-data-table table {\n max-width: unset;\n width: unset;\n}\n</style>\n"
}
},
"a4bf55449be24dfca9a1a95a49a8bd34": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_3a8d1ff4947647e2bd6a128599f2c252",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><h1 id=\"gpt-token-encoder-and-decoder\">GPT token encoder and decoder</h1></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"51bcf572824d4f7cb5c2f044e33b0718": {
"model_module": "jupyter-vuetify",
"model_name": "TextFieldModel",
"model_module_version": "^1.8.5",
"state": {
"_dom_classes": [],
"_events": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_metadata": null,
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.5",
"_model_name": "TextFieldModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.5",
"_view_name": "VuetifyView",
"append_icon": null,
"append_outer_icon": null,
"attributes": {},
"autofocus": null,
"background_color": null,
"children": [],
"class_": null,
"clear_icon": null,
"clearable": null,
"color": null,
"counter": null,
"dark": null,
"dense": null,
"disabled": false,
"error": false,
"error_count": null,
"error_messages": null,
"filled": null,
"flat": null,
"full_width": null,
"height": null,
"hide_details": null,
"hint": null,
"id": null,
"label": "Enter text to tokenize it:",
"layout": null,
"light": null,
"loader_height": null,
"loading": null,
"messages": [],
"outlined": null,
"persistent_hint": null,
"placeholder": null,
"prefix": null,
"prepend_icon": null,
"prepend_inner_icon": null,
"readonly": null,
"reverse": null,
"rounded": null,
"rules": null,
"shaped": null,
"single_line": null,
"slot": null,
"solo": null,
"solo_inverted": null,
"style_": null,
"success": null,
"success_messages": null,
"suffix": null,
"type": null,
"v_model": "Example text is here",
"v_on": null,
"v_slots": [],
"validate_on_blur": null,
"value": null
}
},
"faff82eeaf3448f8aa2450653525fdc1": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_6cb5ff73d0df4c0f8f078ec1bcad7fa7",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><p><span style='font-family: helvetica; color: #CD18FC'>13617</span> <span style='font-family: helvetica; color: #4283FE'>1495</span> <span style='font-family: helvetica; color: #122B59'>374</span> <span style='font-family: helvetica; color: #74BF20'>1618</span></p></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"7ca09aa6ca4e4998846de0b4c9f6423c": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_8a97a4ea57934dd895e196bad61e5792",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><p>4 tokens</p></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"7f7ca25393df4f6f826b54c06a4a8e10": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_36c452da204048f5ac19e32cc531d01d",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><p><span style=\"\n padding: 6px;\n border-right: 3px solid white;\n line-height: 3em;\n font-family: courier;\n background-color: #CD18FC;\n color: white;\n position: relative;\n \"><span style=\"\n position: absolute;\n top: 5.5ch;\n line-height: 1em;\n left: -0.5px;\n font-size: 0.45em\"> 13617</span>Example</span> <span style=\"\n padding: 6px;\n border-right: 3px solid white;\n line-height: 3em;\n font-family: courier;\n background-color: #4283FE;\n color: white;\n position: relative;\n \"><span style=\"\n position: absolute;\n top: 5.5ch;\n line-height: 1em;\n left: -0.5px;\n font-size: 0.45em\"> 1495</span> text</span> <span style=\"\n padding: 6px;\n border-right: 3px solid white;\n line-height: 3em;\n font-family: courier;\n background-color: #122B59;\n color: white;\n position: relative;\n \"><span style=\"\n position: absolute;\n top: 5.5ch;\n line-height: 1em;\n left: -0.5px;\n font-size: 0.45em\"> 374</span> is</span> <span style=\"\n padding: 6px;\n border-right: 3px solid white;\n line-height: 3em;\n font-family: courier;\n background-color: #74BF20;\n color: white;\n position: relative;\n \"><span style=\"\n position: absolute;\n top: 5.5ch;\n line-height: 1em;\n left: -0.5px;\n font-size: 0.45em\"> 1618</span> here</span></p></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"6f3bd51f4df44c3c82d48c2994ad248c": {
"model_module": "jupyter-vuetify",
"model_name": "TextFieldModel",
"model_module_version": "^1.8.5",
"state": {
"_dom_classes": [],
"_events": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_metadata": null,
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.5",
"_model_name": "TextFieldModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.5",
"_view_name": "VuetifyView",
"append_icon": null,
"append_outer_icon": null,
"attributes": {},
"autofocus": null,
"background_color": null,
"children": [],
"class_": null,
"clear_icon": null,
"clearable": null,
"color": null,
"counter": null,
"dark": null,
"dense": null,
"disabled": false,
"error": false,
"error_count": null,
"error_messages": null,
"filled": null,
"flat": null,
"full_width": null,
"height": null,
"hide_details": null,
"hint": null,
"id": null,
"label": "Or convert space separated tokens to text:",
"layout": null,
"light": null,
"loader_height": null,
"loading": null,
"messages": [],
"outlined": null,
"persistent_hint": null,
"placeholder": null,
"prefix": null,
"prepend_icon": null,
"prepend_inner_icon": null,
"readonly": null,
"reverse": null,
"rounded": null,
"rules": null,
"shaped": null,
"single_line": null,
"slot": null,
"solo": null,
"solo_inverted": null,
"style_": null,
"success": null,
"success_messages": null,
"suffix": null,
"type": null,
"v_model": "",
"v_on": null,
"v_slots": [],
"validate_on_blur": null,
"value": null
}
},
"b278f27619374405a45dddc382ecc617": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_97ba1aec3c924319b9cc0fba11138e8a",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"ac3242989ff040ceaf8589e06245da56": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_0986a78a60c944cea088ed86009b07af",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><h2 id=\"search-tokens\">Search tokens</h2></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"dd99642268244a80b0c15467cbbbf11c": {
"model_module": "jupyter-vuetify",
"model_name": "TextFieldModel",
"model_module_version": "^1.8.5",
"state": {
"_dom_classes": [],
"_events": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_metadata": null,
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.5",
"_model_name": "TextFieldModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.5",
"_view_name": "VuetifyView",
"append_icon": null,
"append_outer_icon": null,
"attributes": {},
"autofocus": null,
"background_color": null,
"children": [],
"class_": null,
"clear_icon": null,
"clearable": null,
"color": null,
"counter": null,
"dark": null,
"dense": null,
"disabled": false,
"error": false,
"error_count": null,
"error_messages": null,
"filled": null,
"flat": null,
"full_width": null,
"height": null,
"hide_details": null,
"hint": null,
"id": null,
"label": "Search for a token:",
"layout": null,
"light": null,
"loader_height": null,
"loading": null,
"messages": [],
"outlined": null,
"persistent_hint": null,
"placeholder": null,
"prefix": null,
"prepend_icon": null,
"prepend_inner_icon": null,
"readonly": null,
"reverse": null,
"rounded": null,
"rules": null,
"shaped": null,
"single_line": null,
"slot": null,
"solo": null,
"solo_inverted": null,
"style_": null,
"success": null,
"success_messages": null,
"suffix": null,
"type": null,
"v_model": "",
"v_on": null,
"v_slots": [],
"validate_on_blur": null,
"value": null
}
},
"be21d9a188db45dd9e47184e6669b63d": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"components": null,
"css": null,
"data": null,
"events": [],
"layout": "IPY_MODEL_8670e01752f54508ba5b158ef51cb032",
"methods": null,
"template": "\n<template>\n <div class=\"solara-markdown rendered_html jp-RenderedHTMLCommon\" style=\"\"><p>50,257 results</p></div>\n</template>\n\n<script>\nmodule.exports = {\n mounted() {\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n this.$el.querySelectorAll(\"a\").forEach(a => this.setupRouter(a))\n window.md = this.$el\n },\n methods: {\n setupRouter(a) {\n let href = a.attributes['href'].value;\n if(href.startsWith(\"./\")) {\n // TODO: should we really do this?\n href = location.pathname + href.substr(1);\n a.attributes['href'].href = href;\n // console.log(\"change href to\", href);\n }\n if(href.startsWith(\"./\") || href.startsWith(\"/\")) {\n // console.log(\"connect link with href=\", href, \"to router\")\n a.onclick = e => {\n console.log(\"clicked\", href)\n if(href.startsWith(\"./\")) {\n solara.router.push(href);\n } else {\n solara.router.push(href);\n }\n e.preventDefault()\n }\n } else if(href.startsWith(\"#\")) {\n // console.log(\"connect anchor with href=\", href, \"to custom javascript due to using <base>\")\n href = location.pathname + href;\n a.attributes['href'].value = href;\n } else {\n console.log(\"href\", href, \"is not a local link\")\n }\n }\n },\n updated() {\n // if the html gets update, re-run mermaid\n if(window.mermaid)\n mermaid.init()\n if(window.MathJax && MathJax.Hub) {\n MathJax.Hub.Queue(['Typeset', MathJax.Hub, this.$el]);\n }\n }\n}\n</script>\n "
}
},
"edd1e18c5c5042039453c1766288609c": {
"model_module": "jupyter-vuetify",
"model_name": "VuetifyTemplateModel",
"model_module_version": "^1.8.10",
"state": {
"_component_instances": [],
"_dom_classes": [],
"_jupyter_vue": "IPY_MODEL_386edcd446b843f6a80c5dd6ab0fe816",
"_model_module": "jupyter-vuetify",
"_model_module_version": "^1.8.10",
"_model_name": "VuetifyTemplateModel",
"_view_count": null,
"_view_module": "jupyter-vuetify",
"_view_module_version": "^1.8.10",
"_view_name": "VuetifyView",
"cell_actions": [],
"checked": [],
"column_actions": [],
"column_header_hover": "",
"column_header_widget": null,
"components": null,
"css": null,
"data": null,
"events": [
"on_cell_action",
"on_column_action"
],
"headers": [
{
"text": "token ID",
"value": "token ID",
"sortable": false
},
{
"text": "token",
"value": "token",
"sortable": false
}
],
"headers_selections": [],
"height": null,
"highlighted": null,
"items": [
{
"__row__": 0,
"token ID": "0",
"token": "!"
},
{
"__row__": 1,
"token ID": "1",
"token": "\""
},
{
"__row__": 2,
"token ID": "2",
"token": "#"
},
{
"__row__": 3,
"token ID": "3",
"token": "$"
},
{
"__row__": 4,
"token ID": "4",
"token": "%"
},
{
"__row__": 5,
"token ID": "5",
"token": "&"
},
{
"__row__": 6,
"token ID": "6",
"token": "'"
},
{
"__row__": 7,
"token ID": "7",
"token": "("
},
{
"__row__": 8,
"token ID": "8",
"token": ")"
},
{
"__row__": 9,
"token ID": "9",
"token": "*"
}
],
"items_per_page": 10,
"layout": "IPY_MODEL_07be2cdb751b46faa1ad70006b0e6371",
"methods": null,
"options": {
"page": 1,
"itemsPerPage": 10,
"sortBy": [],
"sortDesc": [],
"groupBy": [],
"groupDesc": [],
"mustSort": false,
"multiSort": false,
"descending": false,
"totalItems": 50257
},
"scrollable": false,
"selection_colors": [],
"selection_enabled": false,
"selections": [],
"template": "IPY_MODEL_78a5a97593ad409095ad5f6edbc4eae6",
"total_length": 50257
}
}
}
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/alonsosilvaallende/218fc71775ea1056e9c0233737dad558/gpt-encoder-decoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "BWPPx2DQl29F"
},
"outputs": [],
"source": [
"!pip install -q solara tiktoken"
]
},
{
"cell_type": "code",
"source": [
"import random\n",
"import solara\n",
"import tiktoken"
],
"metadata": {
"id": "_SfpZ6lPmJKn"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"tokenizer = tiktoken.encoding_for_model(\"gpt-4\")"
],
"metadata": {
"id": "1Rm5RmSAnE6q"
},
"execution_count": 3,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import pandas as pd\n",
"\n",
"df = pd.DataFrame()\n",
"df[\"token ID\"] = range(50257)\n",
"df[\"token\"] = [tokenizer.decode([i]) for i in range(50257)]"
],
"metadata": {
"id": "U3FUzfcjovak"
},
"execution_count": 4,
"outputs": []
},
{
"cell_type": "code",
"source": [
"text1 = solara.reactive(\"Example text is here\")\n",
"text2 = solara.reactive(\"\")\n",
"text3 = solara.reactive(\"\")\n",
"@solara.component\n",
"def Page():\n",
" with solara.Column():\n",
" solara.Markdown(\"#GPT token encoder and decoder\")\n",
" solara.InputText(\"Enter text to tokenize it:\", value=text1, continuous_update=True)\n",
" tokens = tokenizer.encode(text1.value)\n",
" spans1 = \"\"\n",
" spans2 = \"\"\n",
" for i, token in enumerate(tokens):\n",
" random.seed(i)\n",
" random_color = ''.join([random.choice('0123456789ABCDEF') for k in range(6)])\n",
" spans1 += \" \" + f\"<span style='font-family: helvetica; color: #{random_color}'>{token}</span>\"\n",
" spans2 += \" \" + f\"\"\"<span style=\"\n",
" padding: 6px;\n",
" border-right: 3px solid white;\n",
" line-height: 3em;\n",
" font-family: courier;\n",
" background-color: #{random_color};\n",
" color: white;\n",
" position: relative;\n",
" \"><span style=\"\n",
" position: absolute;\n",
" top: 5.5ch;\n",
" line-height: 1em;\n",
" left: -0.5px;\n",
" font-size: 0.45em\"> {token}</span>{tokenizer.decode([token])}</span>\"\"\"\n",
" solara.Markdown(f\"{spans1}\")\n",
" solara.Markdown(f\"{len(tokens)} token\") if len(tokens)==1 else solara.Markdown(f\"{len(tokens)} tokens\")\n",
" solara.Markdown(f'{spans2}')\n",
" solara.InputText(\"Or convert space separated tokens to text:\", value=text2, continuous_update=True)\n",
" token_input = text2.value.split(' ')\n",
" token_input = [int(span) for span in token_input if span != \"\"]\n",
" text_output = tokenizer.decode(token_input)\n",
" solara.Markdown(f'{text_output}')\n",
" solara.Markdown(\"##Search tokens\")\n",
" solara.InputText(\"Search for a token:\", value=text3, continuous_update=True)\n",
" df_subset = df[df[\"token\"].str.startswith(text3.value)]\n",
" solara.Markdown(f\"{df_subset.shape[0]:,} results\")\n",
" solara.DataFrame(df_subset, items_per_page=10)\n",
"\n",
"Page()"
],
"metadata": {
"id": "L-qv-fny2-jo",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 920,
"referenced_widgets": [
"e31fdcbcd4284271a3fc89db929df133",
"fa81ea134de14d9c8945e658f74c8e2c",
"071ed4a34d6a480b9d6a3171c828301a",
"3efd7a6cfdfa44cf91b2a051043478f7",
"3a8d1ff4947647e2bd6a128599f2c252",
"6cb5ff73d0df4c0f8f078ec1bcad7fa7",
"8a97a4ea57934dd895e196bad61e5792",
"36c452da204048f5ac19e32cc531d01d",
"97ba1aec3c924319b9cc0fba11138e8a",
"0986a78a60c944cea088ed86009b07af",
"8670e01752f54508ba5b158ef51cb032",
"07be2cdb751b46faa1ad70006b0e6371",
"78a5a97593ad409095ad5f6edbc4eae6",
"a4bf55449be24dfca9a1a95a49a8bd34",
"51bcf572824d4f7cb5c2f044e33b0718",
"faff82eeaf3448f8aa2450653525fdc1",
"7ca09aa6ca4e4998846de0b4c9f6423c",
"7f7ca25393df4f6f826b54c06a4a8e10",
"6f3bd51f4df44c3c82d48c2994ad248c",
"b278f27619374405a45dddc382ecc617",
"ac3242989ff040ceaf8589e06245da56",
"dd99642268244a80b0c15467cbbbf11c",
"be21d9a188db45dd9e47184e6669b63d",
"edd1e18c5c5042039453c1766288609c"
]
},
"outputId": "fbd67b78-6f45-46d3-95c4-84afd2ed63b6"
},
"execution_count": 5,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"Html(layout=None, style_='display: none', tag='span')"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "e31fdcbcd4284271a3fc89db929df133"
}
},
"metadata": {
"application/vnd.jupyter.widget-view+json": {
"colab": {
"custom_widget_manager": {
"url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/b3e629b1971e1542/manager.min.js"
}
}
}
}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Cannot show ipywidgets in text"
],
"text/html": [
"Cannot show widget. You probably want to rerun the code cell above (<i>Click in the code cell, and press Shift+Enter <kbd>⇧</kbd>+<kbd>↩</kbd></i>)."
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "3efd7a6cfdfa44cf91b2a051043478f7"
}
},
"metadata": {
"application/vnd.jupyter.widget-view+json": {
"colab": {
"custom_widget_manager": {
"url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/b3e629b1971e1542/manager.min.js"
}
}
}
}
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment