Created
February 18, 2022 19:12
-
-
Save jbusecke/b5b940f580714858f39eefbf8ae3a3f8 to your computer and use it in GitHub Desktop.
Snippet to test stuff on LLC4320 in the cloud
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"id": "5f8868c0-1f3f-4864-81cd-00dd8fa98418", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from intake import open_catalog\n", | |
"\n", | |
"cat = open_catalog(\"https://raw.githubusercontent.com/pangeo-data/pangeo-datastore/master/intake-catalogs/ocean/llc4320.yaml\")\n", | |
"ds = cat[\"LLC4320_SST\"].to_dask()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"id": "43596556-9311-4c6c-a9fd-3b7d0e59eb0c", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from dask_gateway import GatewayCluster\n", | |
"from dask.distributed import Client" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"id": "3564405c-7f8f-4ce1-b977-2911e0fa5c9c", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/html": [ | |
"<div>\n", | |
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n", | |
" <div style=\"margin-left: 48px;\">\n", | |
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n", | |
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-46d04569-90ed-11ec-8275-351dd641fa7c</p>\n", | |
" <table style=\"width: 100%; text-align: left;\">\n", | |
"\n", | |
" <tr>\n", | |
" \n", | |
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n", | |
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n", | |
" \n", | |
" </tr>\n", | |
"\n", | |
" \n", | |
" <tr>\n", | |
" <td style=\"text-align: left;\">\n", | |
" <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status\" target=\"_blank\">/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status</a>\n", | |
" </td>\n", | |
" <td style=\"text-align: left;\"></td>\n", | |
" </tr>\n", | |
" \n", | |
"\n", | |
" </table>\n", | |
"\n", | |
" \n", | |
" <details>\n", | |
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n", | |
" <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n", | |
" <h3>GatewayCluster</h3>\n", | |
" <ul>\n", | |
" <li><b>Name: </b>prod.f93b1be7adbd4645b7856abaf95a65ba\n", | |
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status' target='_blank'>/services/dask-gateway/clusters/prod.f93b1be7adbd4645b7856abaf95a65ba/status</a>\n", | |
" </ul>\n", | |
"</div>\n", | |
"\n", | |
" </details>\n", | |
" \n", | |
"\n", | |
" </div>\n", | |
"</div>" | |
], | |
"text/plain": [ | |
"<Client: 'tls://10.8.2.4:8786' processes=0 threads=0, memory=0 B>" | |
] | |
}, | |
"execution_count": 3, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"cluster = GatewayCluster()\n", | |
"cluster.scale(30)\n", | |
"client = Client(cluster)\n", | |
"client" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"id": "45666c87-be91-45da-b636-60ba077e632a", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/html": [ | |
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n", | |
"<defs>\n", | |
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n", | |
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n", | |
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n", | |
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n", | |
"</symbol>\n", | |
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n", | |
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n", | |
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n", | |
"</symbol>\n", | |
"</defs>\n", | |
"</svg>\n", | |
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n", | |
" *\n", | |
" */\n", | |
"\n", | |
":root {\n", | |
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n", | |
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n", | |
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n", | |
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n", | |
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n", | |
" --xr-background-color: var(--jp-layout-color0, white);\n", | |
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n", | |
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", | |
"}\n", | |
"\n", | |
"html[theme=dark],\n", | |
"body.vscode-dark {\n", | |
" --xr-font-color0: rgba(255, 255, 255, 1);\n", | |
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n", | |
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n", | |
" --xr-border-color: #1F1F1F;\n", | |
" --xr-disabled-color: #515151;\n", | |
" --xr-background-color: #111111;\n", | |
" --xr-background-color-row-even: #111111;\n", | |
" --xr-background-color-row-odd: #313131;\n", | |
"}\n", | |
"\n", | |
".xr-wrap {\n", | |
" display: block;\n", | |
" min-width: 300px;\n", | |
" max-width: 700px;\n", | |
"}\n", | |
"\n", | |
".xr-text-repr-fallback {\n", | |
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n", | |
" display: none;\n", | |
"}\n", | |
"\n", | |
".xr-header {\n", | |
" padding-top: 6px;\n", | |
" padding-bottom: 6px;\n", | |
" margin-bottom: 4px;\n", | |
" border-bottom: solid 1px var(--xr-border-color);\n", | |
"}\n", | |
"\n", | |
".xr-header > div,\n", | |
".xr-header > ul {\n", | |
" display: inline;\n", | |
" margin-top: 0;\n", | |
" margin-bottom: 0;\n", | |
"}\n", | |
"\n", | |
".xr-obj-type,\n", | |
".xr-array-name {\n", | |
" margin-left: 2px;\n", | |
" margin-right: 10px;\n", | |
"}\n", | |
"\n", | |
".xr-obj-type {\n", | |
" color: var(--xr-font-color2);\n", | |
"}\n", | |
"\n", | |
".xr-sections {\n", | |
" padding-left: 0 !important;\n", | |
" display: grid;\n", | |
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n", | |
"}\n", | |
"\n", | |
".xr-section-item {\n", | |
" display: contents;\n", | |
"}\n", | |
"\n", | |
".xr-section-item input {\n", | |
" display: none;\n", | |
"}\n", | |
"\n", | |
".xr-section-item input + label {\n", | |
" color: var(--xr-disabled-color);\n", | |
"}\n", | |
"\n", | |
".xr-section-item input:enabled + label {\n", | |
" cursor: pointer;\n", | |
" color: var(--xr-font-color2);\n", | |
"}\n", | |
"\n", | |
".xr-section-item input:enabled + label:hover {\n", | |
" color: var(--xr-font-color0);\n", | |
"}\n", | |
"\n", | |
".xr-section-summary {\n", | |
" grid-column: 1;\n", | |
" color: var(--xr-font-color2);\n", | |
" font-weight: 500;\n", | |
"}\n", | |
"\n", | |
".xr-section-summary > span {\n", | |
" display: inline-block;\n", | |
" padding-left: 0.5em;\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in:disabled + label {\n", | |
" color: var(--xr-font-color2);\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in + label:before {\n", | |
" display: inline-block;\n", | |
" content: '►';\n", | |
" font-size: 11px;\n", | |
" width: 15px;\n", | |
" text-align: center;\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in:disabled + label:before {\n", | |
" color: var(--xr-disabled-color);\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in:checked + label:before {\n", | |
" content: '▼';\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in:checked + label > span {\n", | |
" display: none;\n", | |
"}\n", | |
"\n", | |
".xr-section-summary,\n", | |
".xr-section-inline-details {\n", | |
" padding-top: 4px;\n", | |
" padding-bottom: 4px;\n", | |
"}\n", | |
"\n", | |
".xr-section-inline-details {\n", | |
" grid-column: 2 / -1;\n", | |
"}\n", | |
"\n", | |
".xr-section-details {\n", | |
" display: none;\n", | |
" grid-column: 1 / -1;\n", | |
" margin-bottom: 5px;\n", | |
"}\n", | |
"\n", | |
".xr-section-summary-in:checked ~ .xr-section-details {\n", | |
" display: contents;\n", | |
"}\n", | |
"\n", | |
".xr-array-wrap {\n", | |
" grid-column: 1 / -1;\n", | |
" display: grid;\n", | |
" grid-template-columns: 20px auto;\n", | |
"}\n", | |
"\n", | |
".xr-array-wrap > label {\n", | |
" grid-column: 1;\n", | |
" vertical-align: top;\n", | |
"}\n", | |
"\n", | |
".xr-preview {\n", | |
" color: var(--xr-font-color3);\n", | |
"}\n", | |
"\n", | |
".xr-array-preview,\n", | |
".xr-array-data {\n", | |
" padding: 0 5px !important;\n", | |
" grid-column: 2;\n", | |
"}\n", | |
"\n", | |
".xr-array-data,\n", | |
".xr-array-in:checked ~ .xr-array-preview {\n", | |
" display: none;\n", | |
"}\n", | |
"\n", | |
".xr-array-in:checked ~ .xr-array-data,\n", | |
".xr-array-preview {\n", | |
" display: inline-block;\n", | |
"}\n", | |
"\n", | |
".xr-dim-list {\n", | |
" display: inline-block !important;\n", | |
" list-style: none;\n", | |
" padding: 0 !important;\n", | |
" margin: 0;\n", | |
"}\n", | |
"\n", | |
".xr-dim-list li {\n", | |
" display: inline-block;\n", | |
" padding: 0;\n", | |
" margin: 0;\n", | |
"}\n", | |
"\n", | |
".xr-dim-list:before {\n", | |
" content: '(';\n", | |
"}\n", | |
"\n", | |
".xr-dim-list:after {\n", | |
" content: ')';\n", | |
"}\n", | |
"\n", | |
".xr-dim-list li:not(:last-child):after {\n", | |
" content: ',';\n", | |
" padding-right: 5px;\n", | |
"}\n", | |
"\n", | |
".xr-has-index {\n", | |
" font-weight: bold;\n", | |
"}\n", | |
"\n", | |
".xr-var-list,\n", | |
".xr-var-item {\n", | |
" display: contents;\n", | |
"}\n", | |
"\n", | |
".xr-var-item > div,\n", | |
".xr-var-item label,\n", | |
".xr-var-item > .xr-var-name span {\n", | |
" background-color: var(--xr-background-color-row-even);\n", | |
" margin-bottom: 0;\n", | |
"}\n", | |
"\n", | |
".xr-var-item > .xr-var-name:hover span {\n", | |
" padding-right: 5px;\n", | |
"}\n", | |
"\n", | |
".xr-var-list > li:nth-child(odd) > div,\n", | |
".xr-var-list > li:nth-child(odd) > label,\n", | |
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n", | |
" background-color: var(--xr-background-color-row-odd);\n", | |
"}\n", | |
"\n", | |
".xr-var-name {\n", | |
" grid-column: 1;\n", | |
"}\n", | |
"\n", | |
".xr-var-dims {\n", | |
" grid-column: 2;\n", | |
"}\n", | |
"\n", | |
".xr-var-dtype {\n", | |
" grid-column: 3;\n", | |
" text-align: right;\n", | |
" color: var(--xr-font-color2);\n", | |
"}\n", | |
"\n", | |
".xr-var-preview {\n", | |
" grid-column: 4;\n", | |
"}\n", | |
"\n", | |
".xr-var-name,\n", | |
".xr-var-dims,\n", | |
".xr-var-dtype,\n", | |
".xr-preview,\n", | |
".xr-attrs dt {\n", | |
" white-space: nowrap;\n", | |
" overflow: hidden;\n", | |
" text-overflow: ellipsis;\n", | |
" padding-right: 10px;\n", | |
"}\n", | |
"\n", | |
".xr-var-name:hover,\n", | |
".xr-var-dims:hover,\n", | |
".xr-var-dtype:hover,\n", | |
".xr-attrs dt:hover {\n", | |
" overflow: visible;\n", | |
" width: auto;\n", | |
" z-index: 1;\n", | |
"}\n", | |
"\n", | |
".xr-var-attrs,\n", | |
".xr-var-data {\n", | |
" display: none;\n", | |
" background-color: var(--xr-background-color) !important;\n", | |
" padding-bottom: 5px !important;\n", | |
"}\n", | |
"\n", | |
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n", | |
".xr-var-data-in:checked ~ .xr-var-data {\n", | |
" display: block;\n", | |
"}\n", | |
"\n", | |
".xr-var-data > table {\n", | |
" float: right;\n", | |
"}\n", | |
"\n", | |
".xr-var-name span,\n", | |
".xr-var-data,\n", | |
".xr-attrs {\n", | |
" padding-left: 25px !important;\n", | |
"}\n", | |
"\n", | |
".xr-attrs,\n", | |
".xr-var-attrs,\n", | |
".xr-var-data {\n", | |
" grid-column: 1 / -1;\n", | |
"}\n", | |
"\n", | |
"dl.xr-attrs {\n", | |
" padding: 0;\n", | |
" margin: 0;\n", | |
" display: grid;\n", | |
" grid-template-columns: 125px auto;\n", | |
"}\n", | |
"\n", | |
".xr-attrs dt,\n", | |
".xr-attrs dd {\n", | |
" padding: 0;\n", | |
" margin: 0;\n", | |
" float: left;\n", | |
" padding-right: 10px;\n", | |
" width: auto;\n", | |
"}\n", | |
"\n", | |
".xr-attrs dt {\n", | |
" font-weight: normal;\n", | |
" grid-column: 1;\n", | |
"}\n", | |
"\n", | |
".xr-attrs dt:hover span {\n", | |
" display: inline-block;\n", | |
" background: var(--xr-background-color);\n", | |
" padding-right: 10px;\n", | |
"}\n", | |
"\n", | |
".xr-attrs dd {\n", | |
" grid-column: 2;\n", | |
" white-space: pre-wrap;\n", | |
" word-break: break-all;\n", | |
"}\n", | |
"\n", | |
".xr-icon-database,\n", | |
".xr-icon-file-text2 {\n", | |
" display: inline-block;\n", | |
" vertical-align: middle;\n", | |
" width: 1em;\n", | |
" height: 1.5em !important;\n", | |
" stroke-width: 0;\n", | |
" stroke: currentColor;\n", | |
" fill: currentColor;\n", | |
"}\n", | |
"</style><pre class='xr-text-repr-fallback'><xarray.DataArray 'SST' (time: 600, face: 13, j: 4320, i: 4320)>\n", | |
"dask.array<getitem, shape=(600, 13, 4320, 4320), dtype=float32, chunksize=(1, 1, 4320, 4320), chunktype=numpy.ndarray>\n", | |
"Coordinates:\n", | |
" * face (face) int64 0 1 2 3 4 5 6 7 8 9 10 11 12\n", | |
" * i (i) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n", | |
" * j (j) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n", | |
" * time (time) datetime64[ns] 2011-09-13 ... 2011-10-07T23:00:00</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'SST'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 600</li><li><span class='xr-has-index'>face</span>: 13</li><li><span class='xr-has-index'>j</span>: 4320</li><li><span class='xr-has-index'>i</span>: 4320</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-58f0dc23-0570-4eda-88e3-a3114bfde403' class='xr-array-in' type='checkbox' checked><label for='section-58f0dc23-0570-4eda-88e3-a3114bfde403' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>dask.array<chunksize=(1, 1, 4320, 4320), meta=np.ndarray></span></div><div class='xr-array-data'><table>\n", | |
" <tr>\n", | |
" <td>\n", | |
" <table>\n", | |
" <thead>\n", | |
" <tr>\n", | |
" <td> </td>\n", | |
" <th> Array </th>\n", | |
" <th> Chunk </th>\n", | |
" </tr>\n", | |
" </thead>\n", | |
" <tbody>\n", | |
" \n", | |
" <tr>\n", | |
" <th> Bytes </th>\n", | |
" <td> 542.28 GiB </td>\n", | |
" <td> 71.19 MiB </td>\n", | |
" </tr>\n", | |
" \n", | |
" <tr>\n", | |
" <th> Shape </th>\n", | |
" <td> (600, 13, 4320, 4320) </td>\n", | |
" <td> (1, 1, 4320, 4320) </td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th> Count </th>\n", | |
" <td> 125191 Tasks </td>\n", | |
" <td> 7800 Chunks </td>\n", | |
" </tr>\n", | |
" <tr>\n", | |
" <th> Type </th>\n", | |
" <td> float32 </td>\n", | |
" <td> numpy.ndarray </td>\n", | |
" </tr>\n", | |
" </tbody>\n", | |
" </table>\n", | |
" </td>\n", | |
" <td>\n", | |
" <svg width=\"404\" height=\"184\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n", | |
"\n", | |
" <!-- Horizontal lines -->\n", | |
" <line x1=\"0\" y1=\"0\" x2=\"40\" y2=\"0\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"0\" y1=\"25\" x2=\"40\" y2=\"25\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Vertical lines -->\n", | |
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"25\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"1\" y1=\"0\" x2=\"1\" y2=\"25\" />\n", | |
" <line x1=\"2\" y1=\"0\" x2=\"2\" y2=\"25\" />\n", | |
" <line x1=\"3\" y1=\"0\" x2=\"3\" y2=\"25\" />\n", | |
" <line x1=\"5\" y1=\"0\" x2=\"5\" y2=\"25\" />\n", | |
" <line x1=\"6\" y1=\"0\" x2=\"6\" y2=\"25\" />\n", | |
" <line x1=\"7\" y1=\"0\" x2=\"7\" y2=\"25\" />\n", | |
" <line x1=\"8\" y1=\"0\" x2=\"8\" y2=\"25\" />\n", | |
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"25\" />\n", | |
" <line x1=\"11\" y1=\"0\" x2=\"11\" y2=\"25\" />\n", | |
" <line x1=\"12\" y1=\"0\" x2=\"12\" y2=\"25\" />\n", | |
" <line x1=\"13\" y1=\"0\" x2=\"13\" y2=\"25\" />\n", | |
" <line x1=\"15\" y1=\"0\" x2=\"15\" y2=\"25\" />\n", | |
" <line x1=\"16\" y1=\"0\" x2=\"16\" y2=\"25\" />\n", | |
" <line x1=\"17\" y1=\"0\" x2=\"17\" y2=\"25\" />\n", | |
" <line x1=\"18\" y1=\"0\" x2=\"18\" y2=\"25\" />\n", | |
" <line x1=\"20\" y1=\"0\" x2=\"20\" y2=\"25\" />\n", | |
" <line x1=\"21\" y1=\"0\" x2=\"21\" y2=\"25\" />\n", | |
" <line x1=\"22\" y1=\"0\" x2=\"22\" y2=\"25\" />\n", | |
" <line x1=\"23\" y1=\"0\" x2=\"23\" y2=\"25\" />\n", | |
" <line x1=\"25\" y1=\"0\" x2=\"25\" y2=\"25\" />\n", | |
" <line x1=\"26\" y1=\"0\" x2=\"26\" y2=\"25\" />\n", | |
" <line x1=\"27\" y1=\"0\" x2=\"27\" y2=\"25\" />\n", | |
" <line x1=\"28\" y1=\"0\" x2=\"28\" y2=\"25\" />\n", | |
" <line x1=\"30\" y1=\"0\" x2=\"30\" y2=\"25\" />\n", | |
" <line x1=\"31\" y1=\"0\" x2=\"31\" y2=\"25\" />\n", | |
" <line x1=\"32\" y1=\"0\" x2=\"32\" y2=\"25\" />\n", | |
" <line x1=\"34\" y1=\"0\" x2=\"34\" y2=\"25\" />\n", | |
" <line x1=\"35\" y1=\"0\" x2=\"35\" y2=\"25\" />\n", | |
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"25\" />\n", | |
" <line x1=\"37\" y1=\"0\" x2=\"37\" y2=\"25\" />\n", | |
" <line x1=\"39\" y1=\"0\" x2=\"39\" y2=\"25\" />\n", | |
" <line x1=\"40\" y1=\"0\" x2=\"40\" y2=\"25\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Colored Rectangle -->\n", | |
" <polygon points=\"0.0,0.0 40.328955679569724,0.0 40.328955679569724,25.412616514582485 0.0,25.412616514582485\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n", | |
"\n", | |
" <!-- Text -->\n", | |
" <text x=\"20.164478\" y=\"45.412617\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >600</text>\n", | |
" <text x=\"60.328956\" y=\"12.706308\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,60.328956,12.706308)\">1</text>\n", | |
"\n", | |
"\n", | |
" <!-- Horizontal lines -->\n", | |
" <line x1=\"110\" y1=\"0\" x2=\"124\" y2=\"14\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"110\" y1=\"120\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Vertical lines -->\n", | |
" <line x1=\"110\" y1=\"0\" x2=\"110\" y2=\"120\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"111\" y1=\"1\" x2=\"111\" y2=\"121\" />\n", | |
" <line x1=\"112\" y1=\"2\" x2=\"112\" y2=\"122\" />\n", | |
" <line x1=\"113\" y1=\"3\" x2=\"113\" y2=\"123\" />\n", | |
" <line x1=\"114\" y1=\"4\" x2=\"114\" y2=\"124\" />\n", | |
" <line x1=\"115\" y1=\"5\" x2=\"115\" y2=\"125\" />\n", | |
" <line x1=\"116\" y1=\"6\" x2=\"116\" y2=\"126\" />\n", | |
" <line x1=\"118\" y1=\"8\" x2=\"118\" y2=\"128\" />\n", | |
" <line x1=\"119\" y1=\"9\" x2=\"119\" y2=\"129\" />\n", | |
" <line x1=\"120\" y1=\"10\" x2=\"120\" y2=\"130\" />\n", | |
" <line x1=\"121\" y1=\"11\" x2=\"121\" y2=\"131\" />\n", | |
" <line x1=\"122\" y1=\"12\" x2=\"122\" y2=\"132\" />\n", | |
" <line x1=\"123\" y1=\"13\" x2=\"123\" y2=\"133\" />\n", | |
" <line x1=\"124\" y1=\"14\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Colored Rectangle -->\n", | |
" <polygon points=\"110.0,0.0 124.9485979497544,14.948597949754403 124.9485979497544,134.9485979497544 110.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n", | |
"\n", | |
" <!-- Horizontal lines -->\n", | |
" <line x1=\"110\" y1=\"0\" x2=\"230\" y2=\"0\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"111\" y1=\"1\" x2=\"231\" y2=\"1\" />\n", | |
" <line x1=\"112\" y1=\"2\" x2=\"232\" y2=\"2\" />\n", | |
" <line x1=\"113\" y1=\"3\" x2=\"233\" y2=\"3\" />\n", | |
" <line x1=\"114\" y1=\"4\" x2=\"234\" y2=\"4\" />\n", | |
" <line x1=\"115\" y1=\"5\" x2=\"235\" y2=\"5\" />\n", | |
" <line x1=\"116\" y1=\"6\" x2=\"236\" y2=\"6\" />\n", | |
" <line x1=\"118\" y1=\"8\" x2=\"238\" y2=\"8\" />\n", | |
" <line x1=\"119\" y1=\"9\" x2=\"239\" y2=\"9\" />\n", | |
" <line x1=\"120\" y1=\"10\" x2=\"240\" y2=\"10\" />\n", | |
" <line x1=\"121\" y1=\"11\" x2=\"241\" y2=\"11\" />\n", | |
" <line x1=\"122\" y1=\"12\" x2=\"242\" y2=\"12\" />\n", | |
" <line x1=\"123\" y1=\"13\" x2=\"243\" y2=\"13\" />\n", | |
" <line x1=\"124\" y1=\"14\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Vertical lines -->\n", | |
" <line x1=\"110\" y1=\"0\" x2=\"124\" y2=\"14\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"230\" y1=\"0\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Colored Rectangle -->\n", | |
" <polygon points=\"110.0,0.0 230.0,0.0 244.9485979497544,14.948597949754403 124.9485979497544,14.948597949754403\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n", | |
"\n", | |
" <!-- Horizontal lines -->\n", | |
" <line x1=\"124\" y1=\"14\" x2=\"244\" y2=\"14\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"124\" y1=\"134\" x2=\"244\" y2=\"134\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Vertical lines -->\n", | |
" <line x1=\"124\" y1=\"14\" x2=\"124\" y2=\"134\" style=\"stroke-width:2\" />\n", | |
" <line x1=\"244\" y1=\"14\" x2=\"244\" y2=\"134\" style=\"stroke-width:2\" />\n", | |
"\n", | |
" <!-- Colored Rectangle -->\n", | |
" <polygon points=\"124.9485979497544,14.948597949754403 244.9485979497544,14.948597949754403 244.9485979497544,134.9485979497544 124.9485979497544,134.9485979497544\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n", | |
"\n", | |
" <!-- Text -->\n", | |
" <text x=\"184.948598\" y=\"154.948598\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >4320</text>\n", | |
" <text x=\"264.948598\" y=\"74.948598\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,264.948598,74.948598)\">4320</text>\n", | |
" <text x=\"107.474299\" y=\"147.474299\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,107.474299,147.474299)\">13</text>\n", | |
"</svg>\n", | |
" </td>\n", | |
" </tr>\n", | |
"</table></div></div></li><li class='xr-section-item'><input id='section-aeb3019b-9ab2-40b3-858a-6f1e53ca2da0' class='xr-section-summary-in' type='checkbox' checked><label for='section-aeb3019b-9ab2-40b3-858a-6f1e53ca2da0' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>face</span></div><div class='xr-var-dims'>(face)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 5 6 7 8 9 10 11 12</div><input id='attrs-556c0969-2208-4ae7-a37e-250d40d48394' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-556c0969-2208-4ae7-a37e-250d40d48394' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b966cc8a-5b4a-4a12-8b98-914aa15db15d' class='xr-var-data-in' type='checkbox'><label for='data-b966cc8a-5b4a-4a12-8b98-914aa15db15d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>face_index</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>i</span></div><div class='xr-var-dims'>(i)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 ... 4316 4317 4318 4319</div><input id='attrs-dc8af69b-460c-4c54-b958-890c8d3c4b92' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-dc8af69b-460c-4c54-b958-890c8d3c4b92' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-ea6cc7ff-805b-44d5-8e9d-0783eef0c5af' class='xr-var-data-in' type='checkbox'><label for='data-ea6cc7ff-805b-44d5-8e9d-0783eef0c5af' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>X</dd><dt><span>long_name :</span></dt><dd>x-dimension of the t grid</dd><dt><span>standard_name :</span></dt><dd>x_grid_index</dd><dt><span>swap_dim :</span></dt><dd>XC</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 4317, 4318, 4319])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>j</span></div><div class='xr-var-dims'>(j)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 4 ... 4316 4317 4318 4319</div><input id='attrs-28f93dc6-4444-43ca-b06a-e95e7b2ea642' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-28f93dc6-4444-43ca-b06a-e95e7b2ea642' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-8599b199-9000-41dc-8784-c691798c8220' class='xr-var-data-in' type='checkbox'><label for='data-8599b199-9000-41dc-8784-c691798c8220' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Y</dd><dt><span>long_name :</span></dt><dd>y-dimension of the t grid</dd><dt><span>standard_name :</span></dt><dd>y_grid_index</dd><dt><span>swap_dim :</span></dt><dd>YC</dd></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 4317, 4318, 4319])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2011-09-13 ... 2011-10-07T23:00:00</div><input id='attrs-1daf0bb7-6aff-4fba-8ca6-e0ae92d9c893' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-1daf0bb7-6aff-4fba-8ca6-e0ae92d9c893' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-ed18a344-813a-484a-aebb-3ee60aa24d0a' class='xr-var-data-in' type='checkbox'><label for='data-ed18a344-813a-484a-aebb-3ee60aa24d0a' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>T</dd><dt><span>long_name :</span></dt><dd>Time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array(['2011-09-13T00:00:00.000000000', '2011-09-13T01:00:00.000000000',\n", | |
" '2011-09-13T02:00:00.000000000', ..., '2011-10-07T21:00:00.000000000',\n", | |
" '2011-10-07T22:00:00.000000000', '2011-10-07T23:00:00.000000000'],\n", | |
" dtype='datetime64[ns]')</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-c7de9ac5-bca8-4623-b70b-4ce19abd14aa' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-c7de9ac5-bca8-4623-b70b-4ce19abd14aa' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>" | |
], | |
"text/plain": [ | |
"<xarray.DataArray 'SST' (time: 600, face: 13, j: 4320, i: 4320)>\n", | |
"dask.array<getitem, shape=(600, 13, 4320, 4320), dtype=float32, chunksize=(1, 1, 4320, 4320), chunktype=numpy.ndarray>\n", | |
"Coordinates:\n", | |
" * face (face) int64 0 1 2 3 4 5 6 7 8 9 10 11 12\n", | |
" * i (i) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n", | |
" * j (j) int64 0 1 2 3 4 5 6 7 ... 4313 4314 4315 4316 4317 4318 4319\n", | |
" * time (time) datetime64[ns] 2011-09-13 ... 2011-10-07T23:00:00" | |
] | |
}, | |
"execution_count": 6, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"da = ds.SST.isel(time=slice(0,600))\n", | |
"da" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"id": "3afa77e0-a1e5-4f4c-8c18-355ea0dca293", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import os\n", | |
"PANGEO_SCRATCH = os.environ['PANGEO_SCRATCH']\n", | |
"# -> gs://pangeo-scratch/<username>\n", | |
"import fsspec\n", | |
"mapper = fsspec.get_mapper(f'{PANGEO_SCRATCH}/llc_performance_test/tom_julius_test.zarr')\n", | |
"# mapper can now be to read / write zarr stores" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"id": "e3574ccf-ecf2-4e07-b5c8-c7cca6e68623", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"CPU times: user 10.2 s, sys: 741 ms, total: 11 s\n", | |
"Wall time: 3min 45s\n" | |
] | |
}, | |
{ | |
"data": { | |
"text/plain": [ | |
"<xarray.backends.zarr.ZarrStore at 0x7f59fe4de0b0>" | |
] | |
}, | |
"execution_count": 9, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"%%time\n", | |
"(da+1).to_dataset(name='whatever').to_zarr(mapper)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "f90eb567-66a8-411d-8a79-cf7469e75dbe", | |
"metadata": {}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.9.7" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment