Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/coawst_archive_OSN_make_json_parallel.ipynb

Created September 21, 2023 13:32
Show Gist options
  • Save rsignell-usgs/720c291dc5ed85f70b19c1f862332672 to your computer and use it in GitHub Desktop.
Save rsignell-usgs/720c291dc5ed85f70b19c1f862332672 to your computer and use it in GitHub Desktop.
Download ZIP
coawst_archive_OSN_make_json_parallel.ipynb
Display the source blob
Display the rendered blob
Raw
coawst_archive_OSN_make_json_parallel.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "5b86af71-052b-4081-844b-779db459068d",
"metadata": {},
"source": [
"# COAWST Archive ReferenceFileSystem JSON \n",
"Create ReferenceFileSystem JSON file for a collection of COAWST NetCDF files on S3 "
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "3b6ba617-9772-4ecf-aa7a-2be2a4bcfa1d",
"metadata": {},
"outputs": [
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function now() {\n",
" return new Date();\n",
" }\n",
"\n",
" var force = true;\n",
" var py_version = '3.2.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n",
" var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n",
" var reloading = false;\n",
" var Bokeh = root.Bokeh;\n",
" var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n",
"\n",
" if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n",
" root._bokeh_timeout = Date.now() + 5000;\n",
" root._bokeh_failed_load = false;\n",
" }\n",
"\n",
" function run_callbacks() {\n",
" try {\n",
" root._bokeh_onload_callbacks.forEach(function(callback) {\n",
" if (callback != null)\n",
" callback();\n",
" });\n",
" } finally {\n",
" delete root._bokeh_onload_callbacks;\n",
" }\n",
" console.debug(\"Bokeh: all callbacks have finished\");\n",
" }\n",
"\n",
" function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n",
" if (css_urls == null) css_urls = [];\n",
" if (js_urls == null) js_urls = [];\n",
" if (js_modules == null) js_modules = [];\n",
" if (js_exports == null) js_exports = {};\n",
"\n",
" root._bokeh_onload_callbacks.push(callback);\n",
"\n",
" if (root._bokeh_is_loading > 0) {\n",
" console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
" return null;\n",
" }\n",
" if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n",
" run_callbacks();\n",
" return null;\n",
" }\n",
" if (!reloading) {\n",
" console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
" }\n",
"\n",
" function on_load() {\n",
" root._bokeh_is_loading--;\n",
" if (root._bokeh_is_loading === 0) {\n",
" console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
" run_callbacks()\n",
" }\n",
" }\n",
" window._bokeh_on_load = on_load\n",
"\n",
" function on_error() {\n",
" console.error(\"failed to load \" + url);\n",
" }\n",
"\n",
" var skip = [];\n",
" if (window.requirejs) {\n",
" window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n",
" require([\"jspanel\"], function(jsPanel) {\n",
"\twindow.jsPanel = jsPanel\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-modal\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-tooltip\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-hint\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-layout\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-contextmenu\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"jspanel-dock\"], function() {\n",
"\ton_load()\n",
" })\n",
" require([\"gridstack\"], function(GridStack) {\n",
"\twindow.GridStack = GridStack\n",
"\ton_load()\n",
" })\n",
" require([\"notyf\"], function() {\n",
"\ton_load()\n",
" })\n",
" root._bokeh_is_loading = css_urls.length + 9;\n",
" } else {\n",
" root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n",
" }\n",
"\n",
" var existing_stylesheets = []\n",
" var links = document.getElementsByTagName('link')\n",
" for (var i = 0; i < links.length; i++) {\n",
" var link = links[i]\n",
" if (link.href != null) {\n",
"\texisting_stylesheets.push(link.href)\n",
" }\n",
" }\n",
" for (var i = 0; i < css_urls.length; i++) {\n",
" var url = css_urls[i];\n",
" if (existing_stylesheets.indexOf(url) !== -1) {\n",
"\ton_load()\n",
"\tcontinue;\n",
" }\n",
" const element = document.createElement(\"link\");\n",
" element.onload = on_load;\n",
" element.onerror = on_error;\n",
" element.rel = \"stylesheet\";\n",
" element.type = \"text/css\";\n",
" element.href = url;\n",
" console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
" document.body.appendChild(element);\n",
" } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n",
" var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/dock/jspanel.dock.js'];\n",
" for (var i = 0; i < urls.length; i++) {\n",
" skip.push(urls[i])\n",
" }\n",
" } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n",
" var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/gridstack/[email protected]/dist/gridstack-all.js'];\n",
" for (var i = 0; i < urls.length; i++) {\n",
" skip.push(urls[i])\n",
" }\n",
" } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n",
" var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n",
" for (var i = 0; i < urls.length; i++) {\n",
" skip.push(urls[i])\n",
" }\n",
" } var existing_scripts = []\n",
" var scripts = document.getElementsByTagName('script')\n",
" for (var i = 0; i < scripts.length; i++) {\n",
" var script = scripts[i]\n",
" if (script.src != null) {\n",
"\texisting_scripts.push(script.src)\n",
" }\n",
" }\n",
" for (var i = 0; i < js_urls.length; i++) {\n",
" var url = js_urls[i];\n",
" if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n",
"\tif (!window.requirejs) {\n",
"\t on_load();\n",
"\t}\n",
"\tcontinue;\n",
" }\n",
" var element = document.createElement('script');\n",
" element.onload = on_load;\n",
" element.onerror = on_error;\n",
" element.async = false;\n",
" element.src = url;\n",
" console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
" document.head.appendChild(element);\n",
" }\n",
" for (var i = 0; i < js_modules.length; i++) {\n",
" var url = js_modules[i];\n",
" if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n",
"\tif (!window.requirejs) {\n",
"\t on_load();\n",
"\t}\n",
"\tcontinue;\n",
" }\n",
" var element = document.createElement('script');\n",
" element.onload = on_load;\n",
" element.onerror = on_error;\n",
" element.async = false;\n",
" element.src = url;\n",
" element.type = \"module\";\n",
" console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
" document.head.appendChild(element);\n",
" }\n",
" for (const name in js_exports) {\n",
" var url = js_exports[name];\n",
" if (skip.indexOf(url) >= 0 || root[name] != null) {\n",
"\tif (!window.requirejs) {\n",
"\t on_load();\n",
"\t}\n",
"\tcontinue;\n",
" }\n",
" var element = document.createElement('script');\n",
" element.onerror = on_error;\n",
" element.async = false;\n",
" element.type = \"module\";\n",
" console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
" element.textContent = `\n",
" import ${name} from \"${url}\"\n",
" window.${name} = ${name}\n",
" window._bokeh_on_load()\n",
" `\n",
" document.head.appendChild(element);\n",
" }\n",
" if (!js_urls.length && !js_modules.length) {\n",
" on_load()\n",
" }\n",
" };\n",
"\n",
" function inject_raw_css(css) {\n",
" const element = document.createElement(\"style\");\n",
" element.appendChild(document.createTextNode(css));\n",
" document.body.appendChild(element);\n",
" }\n",
"\n",
" var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.2.2.min.js\", \"https://cdn.holoviz.org/panel/1.2.1/dist/panel.min.js\"];\n",
" var js_modules = [];\n",
" var js_exports = {};\n",
" var css_urls = [];\n",
" var inline_js = [ function(Bokeh) {\n",
" Bokeh.set_log_level(\"info\");\n",
" },\n",
"function(Bokeh) {} // ensure no trailing comma for IE\n",
" ];\n",
"\n",
" function run_inline_js() {\n",
" if ((root.Bokeh !== undefined) || (force === true)) {\n",
" for (var i = 0; i < inline_js.length; i++) {\n",
" inline_js[i].call(root, root.Bokeh);\n",
" }\n",
" // Cache old bokeh versions\n",
" if (Bokeh != undefined && !reloading) {\n",
"\tvar NewBokeh = root.Bokeh;\n",
"\tif (Bokeh.versions === undefined) {\n",
"\t Bokeh.versions = new Map();\n",
"\t}\n",
"\tif (NewBokeh.version !== Bokeh.version) {\n",
"\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n",
"\t}\n",
"\troot.Bokeh = Bokeh;\n",
" }} else if (Date.now() < root._bokeh_timeout) {\n",
" setTimeout(run_inline_js, 100);\n",
" } else if (!root._bokeh_failed_load) {\n",
" console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
" root._bokeh_failed_load = true;\n",
" }\n",
" root._bokeh_is_initializing = false\n",
" }\n",
"\n",
" function load_or_wait() {\n",
" // Implement a backoff loop that tries to ensure we do not load multiple\n",
" // versions of Bokeh and its dependencies at the same time.\n",
" // In recent versions we use the root._bokeh_is_initializing flag\n",
" // to determine whether there is an ongoing attempt to initialize\n",
" // bokeh, however for backward compatibility we also try to ensure\n",
" // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n",
" // before older versions are fully initialized.\n",
" if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n",
" root._bokeh_is_initializing = false;\n",
" root._bokeh_onload_callbacks = undefined;\n",
" console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n",
" load_or_wait();\n",
" } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n",
" setTimeout(load_or_wait, 100);\n",
" } else {\n",
" Bokeh = root.Bokeh;\n",
" bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n",
" root._bokeh_is_initializing = true\n",
" root._bokeh_onload_callbacks = []\n",
" if (!reloading && (!bokeh_loaded || is_dev)) {\n",
"\troot.Bokeh = undefined;\n",
" }\n",
" load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n",
"\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
"\trun_inline_js();\n",
" });\n",
" }\n",
" }\n",
" // Give older versions of the autoload script a head-start to ensure\n",
" // they initialize before we start loading newer version.\n",
" setTimeout(load_or_wait, 100)\n",
"}(window));"
],
"application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.2.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n var reloading = false;\n var Bokeh = root.Bokeh;\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/[email protected]/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/[email protected]/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/gridstack/[email protected]/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.2.2.min.js\", \"https://cdn.holoviz.org/panel/1.2.1/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n Bokeh = root.Bokeh;\n bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));"
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"\n",
"if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n",
" window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n",
"}\n",
"\n",
"\n",
" function JupyterCommManager() {\n",
" }\n",
"\n",
" JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n",
" if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n",
" var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n",
" comm_manager.register_target(comm_id, function(comm) {\n",
" comm.on_msg(msg_handler);\n",
" });\n",
" } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n",
" window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n",
" comm.onMsg = msg_handler;\n",
" });\n",
" } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n",
" google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n",
" var messages = comm.messages[Symbol.asyncIterator]();\n",
" function processIteratorResult(result) {\n",
" var message = result.value;\n",
" console.log(message)\n",
" var content = {data: message.data, comm_id};\n",
" var buffers = []\n",
" for (var buffer of message.buffers || []) {\n",
" buffers.push(new DataView(buffer))\n",
" }\n",
" var metadata = message.metadata || {};\n",
" var msg = {content, buffers, metadata}\n",
" msg_handler(msg);\n",
" return messages.next().then(processIteratorResult);\n",
" }\n",
" return messages.next().then(processIteratorResult);\n",
" })\n",
" }\n",
" }\n",
"\n",
" JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n",
" if (comm_id in window.PyViz.comms) {\n",
" return window.PyViz.comms[comm_id];\n",
" } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n",
" var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n",
" var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n",
" if (msg_handler) {\n",
" comm.on_msg(msg_handler);\n",
" }\n",
" } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n",
" var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n",
" comm.open();\n",
" if (msg_handler) {\n",
" comm.onMsg = msg_handler;\n",
" }\n",
" } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n",
" var comm_promise = google.colab.kernel.comms.open(comm_id)\n",
" comm_promise.then((comm) => {\n",
" window.PyViz.comms[comm_id] = comm;\n",
" if (msg_handler) {\n",
" var messages = comm.messages[Symbol.asyncIterator]();\n",
" function processIteratorResult(result) {\n",
" var message = result.value;\n",
" var content = {data: message.data};\n",
" var metadata = message.metadata || {comm_id};\n",
" var msg = {content, metadata}\n",
" msg_handler(msg);\n",
" return messages.next().then(processIteratorResult);\n",
" }\n",
" return messages.next().then(processIteratorResult);\n",
" }\n",
" }) \n",
" var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n",
" return comm_promise.then((comm) => {\n",
" comm.send(data, metadata, buffers, disposeOnDone);\n",
" });\n",
" };\n",
" var comm = {\n",
" send: sendClosure\n",
" };\n",
" }\n",
" window.PyViz.comms[comm_id] = comm;\n",
" return comm;\n",
" }\n",
" window.PyViz.comm_manager = new JupyterCommManager();\n",
" \n",
"\n",
"\n",
"var JS_MIME_TYPE = 'application/javascript';\n",
"var HTML_MIME_TYPE = 'text/html';\n",
"var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n",
"var CLASS_NAME = 'output';\n",
"\n",
"/**\n",
" * Render data to the DOM node\n",
" */\n",
"function render(props, node) {\n",
" var div = document.createElement(\"div\");\n",
" var script = document.createElement(\"script\");\n",
" node.appendChild(div);\n",
" node.appendChild(script);\n",
"}\n",
"\n",
"/**\n",
" * Handle when a new output is added\n",
" */\n",
"function handle_add_output(event, handle) {\n",
" var output_area = handle.output_area;\n",
" var output = handle.output;\n",
" if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
" return\n",
" }\n",
" var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
" var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
" if (id !== undefined) {\n",
" var nchildren = toinsert.length;\n",
" var html_node = toinsert[nchildren-1].children[0];\n",
" html_node.innerHTML = output.data[HTML_MIME_TYPE];\n",
" var scripts = [];\n",
" var nodelist = html_node.querySelectorAll(\"script\");\n",
" for (var i in nodelist) {\n",
" if (nodelist.hasOwnProperty(i)) {\n",
" scripts.push(nodelist[i])\n",
" }\n",
" }\n",
"\n",
" scripts.forEach( function (oldScript) {\n",
" var newScript = document.createElement(\"script\");\n",
" var attrs = [];\n",
" var nodemap = oldScript.attributes;\n",
" for (var j in nodemap) {\n",
" if (nodemap.hasOwnProperty(j)) {\n",
" attrs.push(nodemap[j])\n",
" }\n",
" }\n",
" attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n",
" newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n",
" oldScript.parentNode.replaceChild(newScript, oldScript);\n",
" });\n",
" if (JS_MIME_TYPE in output.data) {\n",
" toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n",
" }\n",
" output_area._hv_plot_id = id;\n",
" if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n",
" window.PyViz.plot_index[id] = Bokeh.index[id];\n",
" } else {\n",
" window.PyViz.plot_index[id] = null;\n",
" }\n",
" } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
" var bk_div = document.createElement(\"div\");\n",
" bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
" var script_attrs = bk_div.children[0].attributes;\n",
" for (var i = 0; i < script_attrs.length; i++) {\n",
" toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
" }\n",
" // store reference to server id on output_area\n",
" output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
" }\n",
"}\n",
"\n",
"/**\n",
" * Handle when an output is cleared or removed\n",
" */\n",
"function handle_clear_output(event, handle) {\n",
" var id = handle.cell.output_area._hv_plot_id;\n",
" var server_id = handle.cell.output_area._bokeh_server_id;\n",
" if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n",
" var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n",
" if (server_id !== null) {\n",
" comm.send({event_type: 'server_delete', 'id': server_id});\n",
" return;\n",
" } else if (comm !== null) {\n",
" comm.send({event_type: 'delete', 'id': id});\n",
" }\n",
" delete PyViz.plot_index[id];\n",
" if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n",
" var doc = window.Bokeh.index[id].model.document\n",
" doc.clear();\n",
" const i = window.Bokeh.documents.indexOf(doc);\n",
" if (i > -1) {\n",
" window.Bokeh.documents.splice(i, 1);\n",
" }\n",
" }\n",
"}\n",
"\n",
"/**\n",
" * Handle kernel restart event\n",
" */\n",
"function handle_kernel_cleanup(event, handle) {\n",
" delete PyViz.comms[\"hv-extension-comm\"];\n",
" window.PyViz.plot_index = {}\n",
"}\n",
"\n",
"/**\n",
" * Handle update_display_data messages\n",
" */\n",
"function handle_update_output(event, handle) {\n",
" handle_clear_output(event, {cell: {output_area: handle.output_area}})\n",
" handle_add_output(event, handle)\n",
"}\n",
"\n",
"function register_renderer(events, OutputArea) {\n",
" function append_mime(data, metadata, element) {\n",
" // create a DOM node to render to\n",
" var toinsert = this.create_output_subarea(\n",
" metadata,\n",
" CLASS_NAME,\n",
" EXEC_MIME_TYPE\n",
" );\n",
" this.keyboard_manager.register_events(toinsert);\n",
" // Render to node\n",
" var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
" render(props, toinsert[0]);\n",
" element.append(toinsert);\n",
" return toinsert\n",
" }\n",
"\n",
" events.on('output_added.OutputArea', handle_add_output);\n",
" events.on('output_updated.OutputArea', handle_update_output);\n",
" events.on('clear_output.CodeCell', handle_clear_output);\n",
" events.on('delete.Cell', handle_clear_output);\n",
" events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n",
"\n",
" OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
" safe: true,\n",
" index: 0\n",
" });\n",
"}\n",
"\n",
"if (window.Jupyter !== undefined) {\n",
" try {\n",
" var events = require('base/js/events');\n",
" var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
" if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
" register_renderer(events, OutputArea);\n",
" }\n",
" } catch(err) {\n",
" }\n",
"}\n"
],
"application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n"
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<style>*[data-root-id],\n",
"*[data-root-id] > * {\n",
" box-sizing: border-box;\n",
" font-family: var(--jp-ui-font-family);\n",
" font-size: var(--jp-ui-font-size1);\n",
" color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
"}\n",
"\n",
"/* Override VSCode background color */\n",
".cell-output-ipywidget-background:has(\n",
" > .cell-output-ipywidget-background > .lm-Widget > *[data-root-id]\n",
" ),\n",
".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
" background-color: transparent !important;\n",
"}\n",
"</style>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import os\n",
"import fsspec\n",
"import ujson # fast json\n",
"from kerchunk.hdf import SingleHdf5ToZarr \n",
"from kerchunk.combine import MultiZarrToZarr, auto_dask, JustLoad\n",
"from pathlib import Path\n",
"import xarray as xr\n",
"import cf_xarray\n",
"import dask\n",
"import hvplot.xarray"
]
},
{
"cell_type": "markdown",
"id": "647ccdf0-e72c-45c8-8ae5-1ac27c711ab5",
"metadata": {},
"source": [
"#### Start a Dask Gateway cluster"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "7c69bd02-0fcc-4302-b5c6-1b52278768ca",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Region: us-east-1\n",
"No Cluster running.\n",
"Starting new cluster.\n",
"{}\n",
"Setting Cluster Environment Variable AWS_DEFAULT_REGION us-east-1\n",
"Setting Fixed Scaling workers=10\n",
"Reconnect client to clear cache\n",
"client.dashboard_link (for new browser tab/window or dashboard searchbar in Jupyterhub):\n",
"https://nebari.esipfed.org/gateway/clusters/dev.8d52c0d0ee8e46ad83d443fa461ba33e/status\n",
"Propagating environment variables to workers\n",
"Using environment: users/users-pangeo\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"sys.path.append(os.path.join(os.environ['HOME'],'shared','users','lib'))\n",
"import ebdpy as ebd\n",
"\n",
"profile = 'osn-mghp'\n",
"region = 'us-east-1'\n",
"endpoint = 'https://mghp.osn.xsede.org'\n",
"ebd.set_credentials(profile=profile, region=region, endpoint=endpoint)\n",
"worker_max = 10\n",
"client,cluster = ebd.start_dask_cluster(profile=profile, worker_max=worker_max, \n",
" region=region, use_existing_cluster=True,\n",
" adaptive_scaling=False, wait_for_cluster=False, \n",
" worker_profile='Medium Worker', \n",
" propagate_env=True)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "1d1a21f3-9b38-4221-bcab-c118f7917a21",
"metadata": {},
"outputs": [],
"source": [
"fs_read = fsspec.filesystem('s3', anon=True, \n",
" skip_instance_cache=True, \n",
" use_listings_cache=False,\n",
" client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "ab8fba1b-541a-45a3-b091-53fdbed2adf4",
"metadata": {},
"outputs": [],
"source": [
"nc_list = fs_read.glob('s3://rsignellbucket1/COAWST/*.nc')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2ed12a73-7510-4c06-85dd-0b120c98d24d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"rsignellbucket1/COAWST/coawst_us_20220729_01.nc\n",
"rsignellbucket1/COAWST/coawst_us_20230909_13.nc\n"
]
}
],
"source": [
"print(nc_list[0])\n",
"print(nc_list[-1])"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "37cf61de-7fbc-4b72-a877-40a64062051f",
"metadata": {},
"outputs": [],
"source": [
"json_dir = 's3://rsignellbucket1/COAWST/jsons/'"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "858055a4-4963-4c49-9203-0e4f2da3c6b3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"rsignellbucket1/COAWST/jsons/coawst_us_20220729_01.nc.json\n",
"rsignellbucket1/COAWST/jsons/coawst_us_20230508_01.nc.json\n"
]
}
],
"source": [
"json_list = fs_read.glob(f'{json_dir}*.json')\n",
"print(json_list[0])\n",
"print(json_list[-1])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c107fdf8-b3eb-4944-9075-b7b4ef8e59f9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"rsignellbucket1/COAWST/coawst_us_20220729_01.nc\n",
"rsignellbucket1/COAWST/coawst_us_20230508_01.nc\n"
]
}
],
"source": [
"nc_processed_list = [j.split('.json')[0].replace('/jsons','') for j in json_list]\n",
"print(nc_processed_list[0])\n",
"print(nc_processed_list[-1])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "42d69305-3c28-4228-963e-99ac8775dbbc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"241\n",
"rsignellbucket1/COAWST/coawst_us_20230722_01.nc\n",
"rsignellbucket1/COAWST/coawst_us_20230601_13.nc\n"
]
}
],
"source": [
"nc_process_list = list(set(nc_list) - set(nc_processed_list))\n",
"print(len(nc_process_list))\n",
"print(nc_process_list[0])\n",
"print(nc_process_list[-1])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "b9c6b31a-c2b6-48ec-9731-7a2f72a8dfcf",
"metadata": {},
"outputs": [],
"source": [
"fs_write = fsspec.filesystem('s3', anon=False, \n",
" skip_instance_cache=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "1d519baa-d60b-4f41-abec-87aa5a10f21c",
"metadata": {},
"outputs": [],
"source": [
"#for f in flist[:10]:\n",
"# fs.rm(f)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "ff2ea828-1ce7-4781-b73e-fb8d4d4dc46b",
"metadata": {},
"outputs": [],
"source": [
"flist = sorted(['s3://'+f for f in nc_process_list])"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "bcedf939-0b09-4f23-ae79-ffa72d84d03f",
"metadata": {},
"outputs": [],
"source": [
"fsize = [fs_read.size(f) for f in flist]"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "b4a2a398-9814-4d1a-9849-bfa2a9d82761",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"df = pd.DataFrame({'file': flist, 'size': fsize }).sort_values('size')"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "a0c45cf5-0377-4c2c-ba9c-388da7786cf6",
"metadata": {},
"outputs": [],
"source": [
"pd.options.display.max_colwidth=100"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "66a8a122-2f05-40b3-b6d1-8af4474d9305",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>file</th>\n",
" <th>size</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>155</th>\n",
" <td>s3://rsignellbucket1/COAWST/coawst_us_20230726_01.nc</td>\n",
" <td>1924832229</td>\n",
" </tr>\n",
" <tr>\n",
" <th>150</th>\n",
" <td>s3://rsignellbucket1/COAWST/coawst_us_20230723_13.nc</td>\n",
" <td>1925432721</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154</th>\n",
" <td>s3://rsignellbucket1/COAWST/coawst_us_20230725_13.nc</td>\n",
" <td>1925782865</td>\n",
" </tr>\n",
" <tr>\n",
" <th>152</th>\n",
" <td>s3://rsignellbucket1/COAWST/coawst_us_20230724_13.nc</td>\n",
" <td>1925936240</td>\n",
" </tr>\n",
" <tr>\n",
" <th>156</th>\n",
" <td>s3://rsignellbucket1/COAWST/coawst_us_20230726_13.nc</td>\n",
" <td>1926242729</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" file size\n",
"155 s3://rsignellbucket1/COAWST/coawst_us_20230726_01.nc 1924832229\n",
"150 s3://rsignellbucket1/COAWST/coawst_us_20230723_13.nc 1925432721\n",
"154 s3://rsignellbucket1/COAWST/coawst_us_20230725_13.nc 1925782865\n",
"152 s3://rsignellbucket1/COAWST/coawst_us_20230724_13.nc 1925936240\n",
"156 s3://rsignellbucket1/COAWST/coawst_us_20230726_13.nc 1926242729"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.head()"
]
},
{
"cell_type": "markdown",
"id": "6368e297-efa8-41d9-a946-8bac501a3631",
"metadata": {},
"source": [
"Any zero length or small files (e.g. < 1.9GB) above should be transferred again from poseidon to OSN before proceeding:\n",
"\n",
"``` bash\n",
"ssh poseidon.whoi.edu\n",
"cd /vortexfs1/share/usgs-share/Projects/COAWST\n",
"cd 2023\n",
"conda activate aws\n",
"aws s3 cp coawst_us_20230306_13.nc s3://rsignellbucket1/COAWST/ --profile osn-rsignellbucket1 --endpoint-url https://mghp.osn.xsede.org\n",
"```\n",
"Run this notebook again after transfer to make sure all the NetCDF files look okay before creating the JSON files. "
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "754bd605-a604-4556-8c7b-9e30b1901dd1",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"so = dict(mode='rb', anon=False, skip_instance_cache=True,\n",
" client_kwargs={'endpoint_url':'https://mghp.osn.xsede.org'})"
]
},
{
"cell_type": "markdown",
"id": "7c61c32d-f953-43d7-88e6-85b4719349c7",
"metadata": {},
"source": [
"#### Create the individual JSON files directly on S3 "
]
},
{
"cell_type": "markdown",
"id": "c31b04da-7f25-4630-b89d-7074aaa5f7bd",
"metadata": {},
"source": [
"We passed AWS credentials to the Dask workers via environment variables above, and the dask workers don't have the AWS credentials file with profiles defined, so we don't define a profile here, we just set `anon=False` and let the workers find the credentials via the environment variables:"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "cebc056b-f2a2-478e-b368-66f015b2d830",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"json_dir = 's3://rsignellbucket1/COAWST/jsons/'"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "ad2da69d-950a-44ff-a69b-5182d1119cd0",
"metadata": {},
"outputs": [],
"source": [
"# This removes all the JSONs. We wouldn't only do this if we wanted to recreate everything\n",
"#try:\n",
"# fs.rm(json_dir, recursive=True)\n",
"#except:\n",
"# pass"
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "a2aea4a0-c3f2-4811-bc9d-1f9c31177fce",
"metadata": {},
"outputs": [],
"source": [
"def gen_json(u):\n",
" with fs_read.open(u, **so) as infile:\n",
" h5chunks =infile, u, inline_threshold=300)\n",
" fname = Path(u).stem\n",
" outf = f'{json_dir}{fname}.json'\n",
" print(outf)\n",
" with fs_write.open(outf, 'wb') as f:\n",
" f.write(ujson.dumps(h5chunks.translate()).encode());"
]
},
{
"cell_type": "markdown",
"id": "d101115f-cc9f-40a1-85ca-788e298817f1",
"metadata": {},
"source": [
"#### Parallel creation of JSON for each file using Dask Futures"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "519948d6-e291-4350-94d1-7c0e6dc6261f",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"futures = client.map(gen_json, flist)\n",
"client.gather(futures)"
]
},
{
"cell_type": "markdown",
"id": "7df58441-3b29-4eb0-ac2c-763d57deee83",
"metadata": {},
"source": [
"Check disk space on OSN (TB)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "541c21a6-f5cc-4860-a670-0b43ec6dcab3",
"metadata": {},
"outputs": [],
"source": [
"fs_write.du('s3://rsignellbucket1')/1e12"
]
},
{
"cell_type": "markdown",
"id": "f74326ef-2a8e-4515-9acc-f2b526fc367a",
"metadata": {},
"source": [
"#### Switch to a Dask LocalCluster\n",
"Since MultiZarrtoZarr only writes local files, we close the client and create a local cluster instead"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b9a0735c-833e-4ebc-9462-1ab8c3705833",
"metadata": {},
"outputs": [],
"source": [
"#client.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a8cd0ef0-44c5-444d-993d-946959f2203f",
"metadata": {},
"outputs": [],
"source": [
"#from dask.distributed import Client"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f199de5d-3024-475b-951e-c380e758de3f",
"metadata": {},
"outputs": [],
"source": [
"#client = Client(n_workers=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "997b7190-8429-40da-8be0-8b2fe606e4af",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#client"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8725254f-eea3-4191-a2d6-8fa495e3ee2e",
"metadata": {},
"outputs": [],
"source": [
"json_list = fs_read.ls(json_dir)\n",
"json_list = sorted(['s3://'+f for f in json_list])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9255e891-f3fb-43d4-aaf7-b6981c6e8f98",
"metadata": {},
"outputs": [],
"source": [
"opts = dict(anon=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'}, skip_instance_cache=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e35cefea-7d02-44f5-aa1f-a26d4e607be6",
"metadata": {},
"outputs": [],
"source": [
"import base64\n",
"\n",
"import zarr\n",
"import numpy as np\n",
"\n",
"def modify_metadata(out):\n",
" g = zarr.open(out)\n",
" \n",
" fill_value = 1.e+37\n",
" for n, v in g.arrays():\n",
" if len(v.shape) == 4:\n",
" #fill_value = v[-1,-1,0,0]\n",
" v.fill_value = fill_value # 1.e+37\n",
" elif len(v.shape) == 3:\n",
" #fill_value = v[-1,-1,0]\n",
" v.fill_value = fill_value # 1.e+37\n",
" elif len(v.shape) == 2:\n",
" #fill_value = v[-1,-1]\n",
" v.fill_value = fill_value # 1.e+37\n",
" #g[n].fill_value = 1.e+37\n",
" g.ocean_time.attrs['standard_name'] = 'time'\n",
" return out\n",
" \n",
"def postprocess(out):\n",
" out = modify_metadata(out)\n",
" return out"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1150a650-9616-43a2-84ca-a4c056f02d2f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "efc08454-ea63-429c-b956-a98080db512a",
"metadata": {},
"outputs": [],
"source": [
"opts = dict(anon=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})"
]
},
{
"cell_type": "markdown",
"id": "00cbcf39-b339-489c-a98e-82bb95918df9",
"metadata": {},
"source": [
"The cell below is the old serial way to combine JSONs"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63561437-609f-4e3c-809a-5703a96f93d1",
"metadata": {},
"outputs": [],
"source": [
"# %%time\n",
"# mzz = MultiZarrToZarr(json_list, \n",
"# # remote_protocol = 's3',\n",
"# remote_options = opts,\n",
"# target_options = opts,\n",
"# concat_dims = ['ocean_time'],\n",
"# identical_dims=['lat_psi','lat_rho','lat_u','lat_v',\n",
"# 'lon_psi','lon_rho','lon_u','lon_v'],\n",
"# preprocess=postprocess)\n",
"\n",
"# d = mzz.translate()"
]
},
{
"cell_type": "markdown",
"id": "2f129086-c6ef-46e3-8990-ea06b60fd7c7",
"metadata": {},
"source": [
"Combine the jsons using Dask cluster"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6a5578c4-5acb-4065-9822-9ca916f81697",
"metadata": {},
"outputs": [],
"source": [
"mzz_kwargs = dict(concat_dims = ['ocean_time'],\n",
" identical_dims=['lat_psi','lat_rho','lat_u','lat_v',\n",
" 'lon_psi','lon_rho','lon_u','lon_v'],\n",
" preprocess=postprocess)"
]
},
{
"cell_type": "markdown",
"id": "28a79d34-a9eb-42a2-9636-111cceabb980",
"metadata": {},
"source": [
"Update the json_list with the new files and convert to s3 urls"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5f5a5f1e-b686-4f4e-aada-951bc6c9222c",
"metadata": {},
"outputs": [],
"source": [
"json_list = fs_read.glob(f'{json_dir}*.json')\n",
"json_list = [f's3://{j}' for j in json_list]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a5a7d921-f398-49c2-a9be-410fa1fd84e1",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"d = auto_dask(\n",
" json_list,\n",
" single_driver=JustLoad,\n",
" single_kwargs={\"storage_options\": opts},\n",
" mzz_kwargs=mzz_kwargs,\n",
" n_batches=worker_max, # give one batch to each worker\n",
" remote_protocol=\"s3\",\n",
" remote_options=opts\n",
")"
]
},
{
"cell_type": "markdown",
"id": "5ba64f09-5cd2-4057-bac9-653705c384e9",
"metadata": {},
"source": [
"Examine the resulting dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d8e43e7b-b2be-4a8f-a5c7-10b5445f9e89",
"metadata": {},
"outputs": [],
"source": [
"fs5 = fsspec.filesystem(\"reference\", fo=d, target_options=opts,\n",
" remote_protocol='s3', remote_options=opts,\n",
" skip_instance_cache=True)\n",
"m = fs5.get_mapper(\"\")\n",
"ds = xr.open_dataset(m, engine=\"zarr\", chunks={'ocean_time':12}, \n",
" backend_kwargs=dict(consolidated=False))"
]
},
{
"cell_type": "markdown",
"id": "0780aee2-e817-4597-9437-7667e8ed4d9b",
"metadata": {},
"source": [
"Use CF conventions to select times"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0263ac55-b239-4424-a61f-07f13e871d09",
"metadata": {},
"outputs": [],
"source": [
"ds.cf.isel(time=-1)"
]
},
{
"cell_type": "markdown",
"id": "efc7cab7-345e-4af3-a2b5-5e9ed3ce052c",
"metadata": {},
"source": [
"Write combined JSON to S3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fc2d4dea-b332-459d-b23f-130fc8a283c9",
"metadata": {},
"outputs": [],
"source": [
"combined_json = 's3://rsignellbucket1/COAWST/archive.json'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "07cab3c9-83ed-4e8b-b31b-8d9d1f55193e",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"with fs_write.open(combined_json, 'wb') as f:\n",
" f.write(ujson.dumps(d).encode());"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "19ee9e84-8b8d-4c59-b3c5-dc33a3e2822b",
"metadata": {},
"outputs": [],
"source": [
"fs_write.size(combined_json)/1e6 # combined JSON size in MB"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bdcdd78d-4130-46a4-bc7d-befd0048b4bc",
"metadata": {},
"outputs": [],
"source": [
"fs_write.info(combined_json)"
]
},
{
"cell_type": "markdown",
"id": "503a75b0-4577-47d3-9f97-825e83a18776",
"metadata": {},
"source": [
"#### Try opening the consolidated JSON file from S3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8918a33d-c040-42cd-85bc-fe00e2eb11dc",
"metadata": {},
"outputs": [],
"source": [
"# repeating lines from above for convenience in case notebook is started here:\n",
"combined_json = 's3://rsignellbucket1/COAWST/archive.json'\n",
"opts = dict(anon=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f7f580ce-9168-431b-aa40-733f38ab2f96",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"fs5 = fsspec.filesystem(\"reference\", fo=combined_json, target_options=opts,\n",
" remote_protocol='s3', remote_options=opts,\n",
" skip_instance_cache=True)\n",
"m = fs5.get_mapper(\"\")\n",
"ds = xr.open_dataset(m, engine=\"zarr\", chunks={'ocean_time':12}, \n",
" backend_kwargs=dict(consolidated=False))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d045648b-9355-403c-9a01-28e5acf6773d",
"metadata": {},
"outputs": [],
"source": [
"#fs5 = fsspec.filesystem(\"reference\", fo=json_list[-1], target_options=opts,\n",
"# remote_protocol='s3', remote_options=opts)\n",
"#m = fs5.get_mapper(\"\")\n",
"#ds = xr.open_dataset(m, engine=\"zarr\", chunks={'ocean_time':12}, \n",
"# backend_kwargs=dict(consolidated=False))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c615e12-3e9d-4175-8ab2-b512c4b9a221",
"metadata": {},
"outputs": [],
"source": [
"ds.salt"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0ec77e08-9645-416d-a5ec-081ea340fd94",
"metadata": {},
"outputs": [],
"source": [
"hvplot wants coordinate vars to be chunked the same as data vars, so use "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1fa33b23-d0f3-4d09-a595-c6e9dcaf81d5",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"da = ds['temp'][-10:,-1,:,:].load()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc9ccf42-7f45-4d62-9a25-5deea881a7a3",
"metadata": {},
"outputs": [],
"source": [
"lon_name = da.cf['longitude'].name\n",
"lat_name = da.cf['latitude'].name\n",
"\n",
"da.hvplot.quadmesh(x=lon_name, y=lat_name, geo=True, cmap='turbo', tiles='OSM', rasterize=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5ecffc6e-6510-492d-9eff-e2a5a481d1a0",
"metadata": {},
"outputs": [],
"source": [
"da[:,150,150].hvplot(x='ocean_time')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5d3c7d11-8878-401b-a5c3-309f60d1de87",
"metadata": {},
"outputs": [],
"source": [
"client.close(); cluster.shutdown()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9372b00f-99fe-47ab-b059-81a6e3ca4fef",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "users-users-pangeo",
"language": "python",
"name": "conda-env-users-users-pangeo-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"state": {},
"version_major": 2,
"version_minor": 0
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment