Skip to content

Instantly share code, notes, and snippets.

@jbusecke

jbusecke/how_to_update_xarray_properly_pangeo_cloud-2.ipynb

Last active July 26, 2021 22:35
Show Gist options
  • Save jbusecke/d0da9998f19d235852e7a96d484f3f9b to your computer and use it in GitHub Desktop.
Save jbusecke/d0da9998f19d235852e7a96d484f3f9b to your computer and use it in GitHub Desktop.
Download ZIP
Quick example how to work with a newer xarray version in the pangeo cloud
Display the source blob
Display the rendered blob
Raw
how_to_update_xarray_properly_pangeo_cloud-2.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "da3e662a",
"metadata": {},
"source": [
"# How to work with newer xarray versions on pangeo cloud"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "67caa403",
"metadata": {},
"outputs": [],
"source": [
"# rerun the notebook with different versions to test\n",
"target_version = '0.19.0'"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "67749d61",
"metadata": {},
"outputs": [],
"source": [
"# # Execute this cell once and then comment out and restart notebook\n",
"# !pip install xarray=={target_version} --upgrade\n",
"# !pip install cmip6_preprocessing --upgrade"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "2f6328cd",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'0.19.0'"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import xarray as xr\n",
"xr.__version__"
]
},
{
"cell_type": "markdown",
"id": "db40304c",
"metadata": {},
"source": [
"## Install matching xarray version on dask workers and start a cluster\n",
"This follows instructions [here](https://pangeo.io/cloud.html#dask-software-environment)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "90eef12b",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>gateway://traefik-gcp-uscentral1b-staging-dask-gateway.staging:80/staging.76caddfe892c45cbabb6f3077f87972e</li>\n",
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/staging.76caddfe892c45cbabb6f3077f87972e/status' target='_blank'>/services/dask-gateway/clusters/staging.76caddfe892c45cbabb6f3077f87972e/status</a></li>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>0</li>\n",
" <li><b>Cores: </b>0</li>\n",
" <li><b>Memory: </b>0 B</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: 'tls://10.39.232.29:8786' processes=0 threads=0, memory=0 B>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import subprocess\n",
"import logging\n",
"from distributed import WorkerPlugin\n",
"from dask_gateway import GatewayCluster\n",
"from distributed import Client\n",
"\n",
"\n",
"\n",
"class PipPlugin(WorkerPlugin):\n",
" \"\"\"\n",
" Install packages on a worker as it starts up.\n",
"\n",
" Parameters\n",
" ----------\n",
" packages : List[str]\n",
" A list of packages to install with pip on startup.\n",
" \"\"\"\n",
" def __init__(self, packages):\n",
" self.packages = packages\n",
"\n",
" def setup(self, worker):\n",
" logger = logging.getLogger(\"distributed.worker\")\n",
" subprocess.call(['python', '-m', 'pip', 'install', '--upgrade'] + self.packages)\n",
" logger.info(\"Installed %s\", self.packages)\n",
"\n",
"cluster = GatewayCluster()\n",
"client = cluster.get_client()\n",
"\n",
"# if you comment out the following two lines, the compute at the end will break!\n",
"plugin = PipPlugin([f'xarray=={target_version}'])\n",
"client.register_worker_plugin(plugin)\n",
"\n",
"\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "6a9c521a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'tls://10.36.65.5:44121': '0.19.0', 'tls://10.36.67.5:34405': '0.19.0'}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# quickly confirm the versions on the workers\n",
"def check():\n",
" import xarray\n",
" return xarray.__version__\n",
"cluster.scale(4)\n",
"client.wait_for_workers(2)\n",
"client.run(check)"
]
},
{
"cell_type": "markdown",
"id": "c1df6c25",
"metadata": {},
"source": [
"This version needs to match the version above, otherwise you might run into trouble!"
]
},
{
"cell_type": "markdown",
"id": "cac232d8",
"metadata": {},
"source": [
"## Super minimal xarray/dask example"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "96217b1e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;thetao&#x27; (time: 1980, lev: 60)&gt;\n",
"array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)\n",
"Coordinates:\n",
" * lev (lev) float64 5.0 15.0 25.0 35.0 ... 4.875e+03 5.125e+03 5.375e+03\n",
" * time (time) object 1850-01-16 12:00:00 ... 2014-12-16 12:00:00</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'thetao'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 1980</li><li><span class='xr-has-index'>lev</span>: 60</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-060ee6bf-3955-4ee5-a156-4ca54e64663a' class='xr-array-in' type='checkbox' checked><label for='section-060ee6bf-3955-4ee5-a156-4ca54e64663a' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>13.46 13.39 13.24 12.93 12.45 12.1 ... 1.044 1.04 1.009 0.9832 0.9661</span></div><div class='xr-array-data'><pre>array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)</pre></div></div></li><li class='xr-section-item'><input id='section-3d8af7d0-3bad-408a-95f7-f992b991d052' class='xr-section-summary-in' type='checkbox' checked><label for='section-3d8af7d0-3bad-408a-95f7-f992b991d052' class='xr-section-summary' >Coordinates: <span>(2)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lev</span></div><div class='xr-var-dims'>(lev)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>5.0 15.0 ... 5.125e+03 5.375e+03</div><input id='attrs-be711669-4259-4365-a1d3-fd233fda3b5a' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-be711669-4259-4365-a1d3-fd233fda3b5a' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-999252d4-b978-470c-9c72-917e5054ea87' class='xr-var-data-in' type='checkbox'><label for='data-999252d4-b978-470c-9c72-917e5054ea87' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Z</dd><dt><span>bounds :</span></dt><dd>lev_bnds</dd><dt><span>long_name :</span></dt><dd>ocean depth coordinate</dd><dt><span>positive :</span></dt><dd>down</dd><dt><span>realtopology :</span></dt><dd>linear</dd><dt><span>standard_name :</span></dt><dd>depth</dd><dt><span>units :</span></dt><dd>m</dd></dl></div><div class='xr-var-data'><pre>array([5.000000e+00, 1.500000e+01, 2.500000e+01, 3.500000e+01, 4.500000e+01,\n",
" 5.500000e+01, 6.500000e+01, 7.500000e+01, 8.500000e+01, 9.500000e+01,\n",
" 1.050000e+02, 1.150000e+02, 1.250000e+02, 1.350000e+02, 1.450000e+02,\n",
" 1.550000e+02, 1.650984e+02, 1.754790e+02, 1.862912e+02, 1.976603e+02,\n",
" 2.097113e+02, 2.225783e+02, 2.364088e+02, 2.513701e+02, 2.676542e+02,\n",
" 2.854836e+02, 3.051192e+02, 3.268679e+02, 3.510934e+02, 3.782275e+02,\n",
" 4.087846e+02, 4.433777e+02, 4.827367e+02, 5.277280e+02, 5.793729e+02,\n",
" 6.388626e+02, 7.075633e+02, 7.870025e+02, 8.788252e+02, 9.847059e+02,\n",
" 1.106204e+03, 1.244567e+03, 1.400497e+03, 1.573946e+03, 1.764003e+03,\n",
" 1.968944e+03, 2.186457e+03, 2.413972e+03, 2.649001e+03, 2.889385e+03,\n",
" 3.133405e+03, 3.379794e+03, 3.627671e+03, 3.876452e+03, 4.125768e+03,\n",
" 4.375393e+03, 4.625191e+03, 4.875084e+03, 5.125028e+03, 5.375000e+03])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>object</div><div class='xr-var-preview xr-preview'>1850-01-16 12:00:00 ... 2014-12-...</div><input id='attrs-fb7d599e-9cd5-4bc0-803e-910335a94428' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-fb7d599e-9cd5-4bc0-803e-910335a94428' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-ec28dc0c-e697-4400-8830-671b1d74b9df' class='xr-var-data-in' type='checkbox'><label for='data-ec28dc0c-e697-4400-8830-671b1d74b9df' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>T</dd><dt><span>bounds :</span></dt><dd>time_bnds</dd><dt><span>long_name :</span></dt><dd>time</dd><dt><span>realtopology :</span></dt><dd>linear</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array([cftime.DatetimeNoLeap(1850, 1, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 2, 15, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 3, 16, 12, 0, 0, 0), ...,\n",
" cftime.DatetimeNoLeap(2014, 10, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 11, 16, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 12, 16, 12, 0, 0, 0)], dtype=object)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-16f39b6e-7035-4d4d-9097-4bbe31e3c41b' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-16f39b6e-7035-4d4d-9097-4bbe31e3c41b' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'thetao' (time: 1980, lev: 60)>\n",
"array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)\n",
"Coordinates:\n",
" * lev (lev) float64 5.0 15.0 25.0 35.0 ... 4.875e+03 5.125e+03 5.375e+03\n",
" * time (time) object 1850-01-16 12:00:00 ... 2014-12-16 12:00:00"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Super simplistic cloud data example (eliminating intake-esm and cmip6_preprocessing)\n",
"store = 'gs://cmip6/CMIP6/CMIP/E3SM-Project/E3SM-1-0/historical/r3i1p1f1/Omon/thetao/gr/v20200129/'\n",
"ds = xr.open_zarr(store)\n",
"\n",
"# now compute something\n",
"ds.mean(['lon', 'lat']).thetao.load()"
]
},
{
"cell_type": "markdown",
"id": "e07b3192",
"metadata": {},
"source": [
"## Also works with cmip6_pp and intake-esm!"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "5da8ef6a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"--> The keys in the returned dictionary of datasets are constructed as follows:\n",
"\t'activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.zstore.dcpp_init_year.version'\n"
]
},
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='1' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 100.00% [1/1 00:00<00:00]\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;thetao&#x27; (time: 1980, lev: 60)&gt;\n",
"array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)\n",
"Coordinates:\n",
" * lev (lev) float64 5.0 15.0 25.0 35.0 ... 4.875e+03 5.125e+03 5.375e+03\n",
" * time (time) object 1850-01-16 12:00:00 ... 2014-12-16 12:00:00</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'thetao'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 1980</li><li><span class='xr-has-index'>lev</span>: 60</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-4caa0d31-b360-464c-95ca-383bb4ac2616' class='xr-array-in' type='checkbox' checked><label for='section-4caa0d31-b360-464c-95ca-383bb4ac2616' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>13.46 13.39 13.24 12.93 12.45 12.1 ... 1.044 1.04 1.009 0.9832 0.9661</span></div><div class='xr-array-data'><pre>array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)</pre></div></div></li><li class='xr-section-item'><input id='section-dfcf8786-bece-4a7a-9a7b-cbbb3620e8f6' class='xr-section-summary-in' type='checkbox' checked><label for='section-dfcf8786-bece-4a7a-9a7b-cbbb3620e8f6' class='xr-section-summary' >Coordinates: <span>(2)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lev</span></div><div class='xr-var-dims'>(lev)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>5.0 15.0 ... 5.125e+03 5.375e+03</div><input id='attrs-7a1ad455-f722-452b-ab00-94526aee3db1' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-7a1ad455-f722-452b-ab00-94526aee3db1' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-5c4cb7c2-25af-416f-8787-995fc823c9a9' class='xr-var-data-in' type='checkbox'><label for='data-5c4cb7c2-25af-416f-8787-995fc823c9a9' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Z</dd><dt><span>bounds :</span></dt><dd>lev_bnds</dd><dt><span>long_name :</span></dt><dd>ocean depth coordinate</dd><dt><span>positive :</span></dt><dd>down</dd><dt><span>realtopology :</span></dt><dd>linear</dd><dt><span>standard_name :</span></dt><dd>depth</dd><dt><span>units :</span></dt><dd>m</dd></dl></div><div class='xr-var-data'><pre>array([5.000000e+00, 1.500000e+01, 2.500000e+01, 3.500000e+01, 4.500000e+01,\n",
" 5.500000e+01, 6.500000e+01, 7.500000e+01, 8.500000e+01, 9.500000e+01,\n",
" 1.050000e+02, 1.150000e+02, 1.250000e+02, 1.350000e+02, 1.450000e+02,\n",
" 1.550000e+02, 1.650984e+02, 1.754790e+02, 1.862912e+02, 1.976603e+02,\n",
" 2.097113e+02, 2.225783e+02, 2.364088e+02, 2.513701e+02, 2.676542e+02,\n",
" 2.854836e+02, 3.051192e+02, 3.268679e+02, 3.510934e+02, 3.782275e+02,\n",
" 4.087846e+02, 4.433777e+02, 4.827367e+02, 5.277280e+02, 5.793729e+02,\n",
" 6.388626e+02, 7.075633e+02, 7.870025e+02, 8.788252e+02, 9.847059e+02,\n",
" 1.106204e+03, 1.244567e+03, 1.400497e+03, 1.573946e+03, 1.764003e+03,\n",
" 1.968944e+03, 2.186457e+03, 2.413972e+03, 2.649001e+03, 2.889385e+03,\n",
" 3.133405e+03, 3.379794e+03, 3.627671e+03, 3.876452e+03, 4.125768e+03,\n",
" 4.375393e+03, 4.625191e+03, 4.875084e+03, 5.125028e+03, 5.375000e+03])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>object</div><div class='xr-var-preview xr-preview'>1850-01-16 12:00:00 ... 2014-12-...</div><input id='attrs-d6e5af47-dd8c-415f-b3b0-34cbe8a98a58' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-d6e5af47-dd8c-415f-b3b0-34cbe8a98a58' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-d71fad3e-636c-40cb-8f4c-5e5000703439' class='xr-var-data-in' type='checkbox'><label for='data-d71fad3e-636c-40cb-8f4c-5e5000703439' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>T</dd><dt><span>bounds :</span></dt><dd>time_bnds</dd><dt><span>long_name :</span></dt><dd>time</dd><dt><span>realtopology :</span></dt><dd>linear</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array([cftime.DatetimeNoLeap(1850, 1, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 2, 15, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 3, 16, 12, 0, 0, 0), ...,\n",
" cftime.DatetimeNoLeap(2014, 10, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 11, 16, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 12, 16, 12, 0, 0, 0)], dtype=object)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-cfa4681f-3edf-447f-9c15-06737a5ed176' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-cfa4681f-3edf-447f-9c15-06737a5ed176' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'thetao' (time: 1980, lev: 60)>\n",
"array([[13.4562645 , 13.393538 , 13.235398 , ..., 0.9334626 ,\n",
" 0.90995324, 0.9026728 ],\n",
" [13.619133 , 13.573829 , 13.468156 , ..., 0.93346804,\n",
" 0.91000783, 0.902723 ],\n",
" [13.543202 , 13.505928 , 13.432397 , ..., 0.933512 ,\n",
" 0.91005 , 0.902773 ],\n",
" ...,\n",
" [13.882875 , 13.848218 , 13.763058 , ..., 1.0088891 ,\n",
" 0.9831473 , 0.9660718 ],\n",
" [13.666238 , 13.620644 , 13.528952 , ..., 1.0089248 ,\n",
" 0.9831997 , 0.9661009 ],\n",
" [13.815262 , 13.749072 , 13.5942955 , ..., 1.0088788 ,\n",
" 0.98319495, 0.9661138 ]], dtype=float32)\n",
"Coordinates:\n",
" * lev (lev) float64 5.0 15.0 25.0 35.0 ... 4.875e+03 5.125e+03 5.375e+03\n",
" * time (time) object 1850-01-16 12:00:00 ... 2014-12-16 12:00:00"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# also works with cmip6_preprocessing and intake-esm\n",
"import intake\n",
"from cmip6_preprocessing.preprocessing import combined_preprocessing\n",
"\n",
"col = intake.open_esm_datastore(\n",
" \"https://storage.googleapis.com/cmip6/pangeo-cmip6.json\"\n",
")\n",
"\n",
"z_kwargs = {\"consolidated\": True, \"use_cftime\": True}\n",
"query = dict(\n",
" experiment_id=[\"historical\"],\n",
" table_id=[\"Omon\"],\n",
" variable_id=['thetao'],\n",
" member_id=['r3i1p1f1'],\n",
" grid_label=[\"gr\"],\n",
" source_id=[\n",
" \"E3SM-1-0\",\n",
" ],\n",
")\n",
"dset_dict = col.search(**query).to_dataset_dict(\n",
" zarr_kwargs=z_kwargs,\n",
" storage_options={\"token\": \"anon\"},\n",
" preprocess=combined_preprocessing,\n",
" aggregate=False\n",
")\n",
"ds = dset_dict['CMIP.E3SM-Project.E3SM-1-0.historical.r3i1p1f1.Omon.thetao.gr.gs://cmip6/CMIP6/CMIP/E3SM-Project/E3SM-1-0/historical/r3i1p1f1/Omon/thetao/gr/v20200129/.nan.20200129']\n",
"# now compute something\n",
"ds.mean(['x', 'y']).thetao.load()"
]
},
{
"cell_type": "markdown",
"id": "9205a592",
"metadata": {},
"source": [
"I have tested this with `0.19.0` and both work. Once I comment out the lines mentioned in the cluster configuration, both will fail!"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f76b6fae",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment