Skip to content

Instantly share code, notes, and snippets.

@jbusecke
Created February 18, 2022 19:12
Show Gist options
  • Save jbusecke/b5b940f580714858f39eefbf8ae3a3f8 to your computer and use it in GitHub Desktop.
Save jbusecke/b5b940f580714858f39eefbf8ae3a3f8 to your computer and use it in GitHub Desktop.
Snippet to test stuff on LLC4320 in the cloud
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "5f8868c0-1f3f-4864-81cd-00dd8fa98418",
"metadata": {},
"outputs": [],
"source": [
"from intake import open_catalog\n",
"\n",
"cat = open_catalog(\"https://raw.githubusercontent.com/pangeo-data/pangeo-datastore/master/intake-catalogs/ocean/llc4320.yaml\")\n",
"ds = cat[\"LLC4320_SST\"].to_dask()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "43596556-9311-4c6c-a9fd-3b7d0e59eb0c",
"metadata": {},
"outputs": [],
"source": [
"from dask_gateway import GatewayCluster\n",
"from dask.distributed import Client"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "3564405c-7f8f-4ce1-b977-2911e0fa5c9c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-46d04569-90ed-11ec-8275-351dd641fa7c</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status\" target=\"_blank\">/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
" <h3>GatewayCluster</h3>\n",
" <ul>\n",
" <li><b>Name: </b>prod.f93b1be7adbd4645b7856abaf95a65ba\n",
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status' target='_blank'>/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status</a>\n",
" </ul>\n",
"</div>\n",
"\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"text/plain": [
"<Client: 'tls://10.8.2.4:8786' processes=0 threads=0, memory=0 B>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cluster = GatewayCluster()\n",
"cluster.scale(30)\n",
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "45666c87-be91-45da-b636-60ba077e632a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2 {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;SST&#x27; (time: 600, face: 13, j: 4320, i: 4320)&gt;\n",
"dask.array&lt;getitem, shape=(600, 13, 4320, 4320), dtype=float32, chunksize=(1, 1, 4320, 4320), chunktype=numpy.ndarray&gt;\n",
"Coordinates:\n",
" * face (face) int64 0 1 2 3 4 5 6 7 8 9 10 11 12\n",
" * i (i) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n",
" * j (j) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n",
" * time (time) datetime64[ns] 2011-09-13 ... 2011-10-07T23:00:00</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'SST'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 600</li><li><span class='xr-has-index'>face</span>: 13</li><li><span class='xr-has-index'>j</span>: 4320</li><li><span class='xr-has-index'>i</span>: 4320</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-58f0dc23-0570-4eda-88e3-a3114bfde403' class='xr-array-in' type='checkbox' checked><label for='section-58f0dc23-0570-4eda-88e3-a3114bfde403' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>dask.array&lt;chunksize=(1, 1, 4320, 4320), meta=np.ndarray&gt;</span></div><div class='xr-array-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table>\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 542.28 GiB </td>\n",
" <td> 71.19 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (600, 13, 4320, 4320) </td>\n",
" <td> (1, 1, 4320, 4320) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Count </th>\n",
" <td> 125191 Tasks </td>\n",
" <td> 7800 Chunks </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Type </th>\n",
" <td> float32 </td>\n",
" <td> numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"404\" height=\"184\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"40\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"25\" x2=\"40\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"25\" style=\"stroke-width:2\" />\n",
" <line x1=\"1\" y1=\"0\" x2=\"1\" y2=\"25\" />\n",
" <line x1=\"2\" y1=\"0\" x2=\"2\" y2=\"25\" />\n",
" <line x1=\"3\" y1=\"0\" x2=\"3\" y2=\"25\" />\n",
" <line x1=\"5\" y1=\"0\" x2=\"5\" y2=\"25\" />\n",
" <line x1=\"6\" y1=\"0\" x2=\"6\" y2=\"25\" />\n",
" <line x1=\"7\" y1=\"0\" x2=\"7\" y2=\"25\" />\n",
" <line x1=\"8\" y1=\"0\" x2=\"8\" y2=\"25\" />\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"25\" />\n",
" <line x1=\"11\" y1=\"0\" x2=\"11\" y2=\"25\" />\n",
" <line x1=\"12\" y1=\"0\" x2=\"12\" y2=\"25\" />\n",
" <line x1=\"13\" y1=\"0\" x2=\"13\" y2=\"25\" />\n",
" <line x1=\"15\" y1=\"0\" x2=\"15\" y2=\"25\" />\n",
" <line x1=\"16\" y1=\"0\" x2=\"16\" y2=\"25\" />\n",
" <line x1=\"17\" y1=\"0\" x2=\"17\" y2=\"25\" />\n",
" <line x1=\"18\" y1=\"0\" x2=\"18\" y2=\"25\" />\n",
" <line x1=\"20\" y1=\"0\" x2=\"20\" y2=\"25\" />\n",
" <line x1=\"21\" y1=\"0\" x2=\"21\" y2=\"25\" />\n",
" <line x1=\"22\" y1=\"0\" x2=\"22\" y2=\"25\" />\n",
" <line x1=\"23\" y1=\"0\" x2=\"23\" y2=\"25\" />\n",
" <line x1=\"25\" y1=\"0\" x2=\"25\" y2=\"25\" />\n",
" <line x1=\"26\" y1=\"0\" x2=\"26\" y2=\"25\" />\n",
" <line x1=\"27\" y1=\"0\" x2=\"27\" y2=\"25\" />\n",
" <line x1=\"28\" y1=\"0\" x2=\"28\" y2=\"25\" />\n",
" <line x1=\"30\" y1=\"0\" x2=\"30\" y2=\"25\" />\n",
" <line x1=\"31\" y1=\"0\" x2=\"31\" y2=\"25\" />\n",
" <line x1=\"32\" y1=\"0\" x2=\"32\" y2=\"25\" />\n",
" <line x1=\"34\" y1=\"0\" x2=\"34\" y2=\"25\" />\n",
" <line x1=\"35\" y1=\"0\" x2=\"35\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"25\" />\n",
" <line x1=\"37\" y1=\"0\" x2=\"37\" y2=\"25\" />\n",
" <line x1=\"39\" y1=\"0\" x2=\"39\" y2=\"25\" />\n",
" <line x1=\"40\" y1=\"0\" x2=\"40\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.0,0.0 40.328955679569724,0.0 40.328955679569724,25.412616514582485 0.0,25.412616514582485\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"20.164478\" y=\"45.412617\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >600</text>\n",
" <text x=\"60.328956\" y=\"12.706308\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,60.328956,12.706308)\">1</text>\n",
"\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"124\" y2=\"14\" style=\"stroke-width:2\" />\n",
" <line x1=\"110\" y1=\"120\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"110\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"111\" y1=\"1\" x2=\"111\" y2=\"121\" />\n",
" <line x1=\"112\" y1=\"2\" x2=\"112\" y2=\"122\" />\n",
" <line x1=\"113\" y1=\"3\" x2=\"113\" y2=\"123\" />\n",
" <line x1=\"114\" y1=\"4\" x2=\"114\" y2=\"124\" />\n",
" <line x1=\"115\" y1=\"5\" x2=\"115\" y2=\"125\" />\n",
" <line x1=\"116\" y1=\"6\" x2=\"116\" y2=\"126\" />\n",
" <line x1=\"118\" y1=\"8\" x2=\"118\" y2=\"128\" />\n",
" <line x1=\"119\" y1=\"9\" x2=\"119\" y2=\"129\" />\n",
" <line x1=\"120\" y1=\"10\" x2=\"120\" y2=\"130\" />\n",
" <line x1=\"121\" y1=\"11\" x2=\"121\" y2=\"131\" />\n",
" <line x1=\"122\" y1=\"12\" x2=\"122\" y2=\"132\" />\n",
" <line x1=\"123\" y1=\"13\" x2=\"123\" y2=\"133\" />\n",
" <line x1=\"124\" y1=\"14\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.0,0.0 124.9485979497544,14.948597949754403 124.9485979497544,134.9485979497544 110.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"230\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"111\" y1=\"1\" x2=\"231\" y2=\"1\" />\n",
" <line x1=\"112\" y1=\"2\" x2=\"232\" y2=\"2\" />\n",
" <line x1=\"113\" y1=\"3\" x2=\"233\" y2=\"3\" />\n",
" <line x1=\"114\" y1=\"4\" x2=\"234\" y2=\"4\" />\n",
" <line x1=\"115\" y1=\"5\" x2=\"235\" y2=\"5\" />\n",
" <line x1=\"116\" y1=\"6\" x2=\"236\" y2=\"6\" />\n",
" <line x1=\"118\" y1=\"8\" x2=\"238\" y2=\"8\" />\n",
" <line x1=\"119\" y1=\"9\" x2=\"239\" y2=\"9\" />\n",
" <line x1=\"120\" y1=\"10\" x2=\"240\" y2=\"10\" />\n",
" <line x1=\"121\" y1=\"11\" x2=\"241\" y2=\"11\" />\n",
" <line x1=\"122\" y1=\"12\" x2=\"242\" y2=\"12\" />\n",
" <line x1=\"123\" y1=\"13\" x2=\"243\" y2=\"13\" />\n",
" <line x1=\"124\" y1=\"14\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"110\" y1=\"0\" x2=\"124\" y2=\"14\" style=\"stroke-width:2\" />\n",
" <line x1=\"230\" y1=\"0\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"110.0,0.0 230.0,0.0 244.9485979497544,14.948597949754403 124.9485979497544,14.948597949754403\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"124\" y1=\"14\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n",
" <line x1=\"124\" y1=\"134\" x2=\"244\" y2=\"134\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"124\" y1=\"14\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n",
" <line x1=\"244\" y1=\"14\" x2=\"244\" y2=\"134\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"124.9485979497544,14.948597949754403 244.9485979497544,14.948597949754403 244.9485979497544,134.9485979497544 124.9485979497544,134.9485979497544\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"184.948598\" y=\"154.948598\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >4320</text>\n",
" <text x=\"264.948598\" y=\"74.948598\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,264.948598,74.948598)\">4320</text>\n",
" <text x=\"107.474299\" y=\"147.474299\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,107.474299,147.474299)\">13</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></div></li><li class='xr-section-item'><input id='section-aeb3019b-9ab2-40b3-858a-6f1e53ca2da0' class='xr-section-summary-in' type='checkbox' checked><label for='section-aeb3019b-9ab2-40b3-858a-6f1e53ca2da0' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>face</span></div><div class='xr-var-dims'>(face)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 5 6 7 8 9 10 11 12</div><input id='attrs-556c0969-2208-4ae7-a37e-250d40d48394' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-556c0969-2208-4ae7-a37e-250d40d48394' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b966cc8a-5b4a-4a12-8b98-914aa15db15d' class='xr-var-data-in' type='checkbox'><label for='data-b966cc8a-5b4a-4a12-8b98-914aa15db15d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>face_index</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>i</span></div><div class='xr-var-dims'>(i)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 ... 4316 4317 4318 4319</div><input id='attrs-dc8af69b-460c-4c54-b958-890c8d3c4b92' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-dc8af69b-460c-4c54-b958-890c8d3c4b92' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-ea6cc7ff-805b-44d5-8e9d-0783eef0c5af' class='xr-var-data-in' type='checkbox'><label for='data-ea6cc7ff-805b-44d5-8e9d-0783eef0c5af' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>X</dd><dt><span>long_name :</span></dt><dd>x-dimension of the t grid</dd><dt><span>standard_name :</span></dt><dd>x_grid_index</dd><dt><span>swap_dim :</span></dt><dd>XC</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 4317, 4318, 4319])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>j</span></div><div class='xr-var-dims'>(j)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 ... 4316 4317 4318 4319</div><input id='attrs-28f93dc6-4444-43ca-b06a-e95e7b2ea642' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-28f93dc6-4444-43ca-b06a-e95e7b2ea642' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-8599b199-9000-41dc-8784-c691798c8220' class='xr-var-data-in' type='checkbox'><label for='data-8599b199-9000-41dc-8784-c691798c8220' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Y</dd><dt><span>long_name :</span></dt><dd>y-dimension of the t grid</dd><dt><span>standard_name :</span></dt><dd>y_grid_index</dd><dt><span>swap_dim :</span></dt><dd>YC</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 4317, 4318, 4319])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2011-09-13 ... 2011-10-07T23:00:00</div><input id='attrs-1daf0bb7-6aff-4fba-8ca6-e0ae92d9c893' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-1daf0bb7-6aff-4fba-8ca6-e0ae92d9c893' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-ed18a344-813a-484a-aebb-3ee60aa24d0a' class='xr-var-data-in' type='checkbox'><label for='data-ed18a344-813a-484a-aebb-3ee60aa24d0a' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>T</dd><dt><span>long_name :</span></dt><dd>Time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array([&#x27;2011-09-13T00:00:00.000000000&#x27;, &#x27;2011-09-13T01:00:00.000000000&#x27;,\n",
" &#x27;2011-09-13T02:00:00.000000000&#x27;, ..., &#x27;2011-10-07T21:00:00.000000000&#x27;,\n",
" &#x27;2011-10-07T22:00:00.000000000&#x27;, &#x27;2011-10-07T23:00:00.000000000&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-c7de9ac5-bca8-4623-b70b-4ce19abd14aa' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-c7de9ac5-bca8-4623-b70b-4ce19abd14aa' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'SST' (time: 600, face: 13, j: 4320, i: 4320)>\n",
"dask.array<getitem, shape=(600, 13, 4320, 4320), dtype=float32, chunksize=(1, 1, 4320, 4320), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
" * face (face) int64 0 1 2 3 4 5 6 7 8 9 10 11 12\n",
" * i (i) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n",
" * j (j) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n",
" * time (time) datetime64[ns] 2011-09-13 ... 2011-10-07T23:00:00"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"da = ds.SST.isel(time=slice(0,600))\n",
"da"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "3afa77e0-a1e5-4f4c-8c18-355ea0dca293",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"PANGEO_SCRATCH = os.environ['PANGEO_SCRATCH']\n",
"# -> gs://pangeo-scratch/<username>\n",
"import fsspec\n",
"mapper = fsspec.get_mapper(f'{PANGEO_SCRATCH}/llc_performance_test/tom_julius_test.zarr')\n",
"# mapper can now be to read / write zarr stores"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "e3574ccf-ecf2-4e07-b5c8-c7cca6e68623",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 10.2 s, sys: 741 ms, total: 11 s\n",
"Wall time: 3min 45s\n"
]
},
{
"data": {
"text/plain": [
"<xarray.backends.zarr.ZarrStore at 0x7f59fe4de0b0>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"(da+1).to_dataset(name='whatever').to_zarr(mapper)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f90eb567-66a8-411d-8a79-cf7469e75dbe",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment