Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/Big_NetCDF4_on_S3.ipynb

Created August 3, 2023 16:25
Show Gist options
  • Save rsignell-usgs/90d05ddf5f06c23ea9edb347034c1483 to your computer and use it in GitHub Desktop.
Save rsignell-usgs/90d05ddf5f06c23ea9edb347034c1483 to your computer and use it in GitHub Desktop.
Download ZIP
Big_NetCDF4_on_S3.ipynb
Display the source blob
Display the rendered blob
Raw
Big_NetCDF4_on_S3.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "415d30b2-f97b-4160-b75d-9f36140b4cbb",
"metadata": {},
"source": [
"# Opening a big (25GB) NetCDF4 file on S3 with fsspec/xarray\n",
"Just as when opening from a filesystem, when opening from s3, xarray loads:\n",
"* metadata (including coordinate variables) eagerly\n",
"* data variables lazily\n",
"\n",
"We demonstrate this here opening a 25GB NetCDF file from s3 in only 6 seconds. The memory usage in the Notebook shows less than 1GB RAM usage. "
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "c577e09c-0cb2-493b-a8e2-8f4ef911b403",
"metadata": {},
"outputs": [],
"source": [
"import fsspec\n",
"import xarray as xr"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "44b5cbfe-0851-4bc7-ac4f-c0fa80fd8d63",
"metadata": {},
"outputs": [],
"source": [
"fs = fsspec.filesystem('s3', anon=True, client_kwargs=dict(endpoint_url='https://ncsa.osn.xsede.org'))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "bd98b3d5-d00c-42b5-a527-d5e0f470ad11",
"metadata": {},
"outputs": [],
"source": [
"ncfile_on_s3 = 's3://esip/examples/adcirc/adcirc_01.nc'"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "600839af-4ce5-424f-8821-c4d46b3da44b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"26.140007264"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fs.size(ncfile_on_s3)/1e9 # GB"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "389631ff-744e-459a-8ac6-9347f6996aef",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 3.22 s, sys: 292 ms, total: 3.51 s\n",
"Wall time: 9.28 s\n"
]
}
],
"source": [
"%%time\n",
"ds = xr.open_dataset(fs.open(ncfile_on_s3), chunks={'time':10, 'node':141973})"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "44d4345b-8714-4b9d-af87-8e258ddcc158",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;zeta&#x27; (time: 720, node: 9228245)&gt;\n",
"dask.array&lt;open_dataset-07e4e31a5b972cd686bab63d3a63abe9zeta, shape=(720, 9228245), dtype=float64, chunksize=(10, 141973), chunktype=numpy.ndarray&gt;\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 2031-08-03T02:10:00 ... 2031-08-08T02:00:00\n",
" x (node) float64 dask.array&lt;chunksize=(141973,), meta=np.ndarray&gt;\n",
" y (node) float64 dask.array&lt;chunksize=(141973,), meta=np.ndarray&gt;\n",
"Dimensions without coordinates: node\n",
"Attributes:\n",
" location: node\n",
" long_name: water surface elevation above geoid\n",
" mesh: adcirc_mesh\n",
" standard_name: sea_surface_height_above_geoid\n",
" units: m</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'zeta'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 720</li><li><span>node</span>: 9228245</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-80b72be7-7a05-4bd7-bd6c-40a2f12afc8d' class='xr-array-in' type='checkbox' checked><label for='section-80b72be7-7a05-4bd7-bd6c-40a2f12afc8d' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>dask.array&lt;chunksize=(10, 141973), meta=np.ndarray&gt;</span></div><div class='xr-array-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 49.50 GiB </td>\n",
" <td> 10.83 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (720, 9228245) </td>\n",
" <td> (10, 141973) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 4680 chunks in 2 graph layers </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"170\" height=\"75\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"1\" x2=\"120\" y2=\"1\" />\n",
" <line x1=\"0\" y1=\"2\" x2=\"120\" y2=\"2\" />\n",
" <line x1=\"0\" y1=\"3\" x2=\"120\" y2=\"3\" />\n",
" <line x1=\"0\" y1=\"5\" x2=\"120\" y2=\"5\" />\n",
" <line x1=\"0\" y1=\"6\" x2=\"120\" y2=\"6\" />\n",
" <line x1=\"0\" y1=\"7\" x2=\"120\" y2=\"7\" />\n",
" <line x1=\"0\" y1=\"9\" x2=\"120\" y2=\"9\" />\n",
" <line x1=\"0\" y1=\"10\" x2=\"120\" y2=\"10\" />\n",
" <line x1=\"0\" y1=\"12\" x2=\"120\" y2=\"12\" />\n",
" <line x1=\"0\" y1=\"13\" x2=\"120\" y2=\"13\" />\n",
" <line x1=\"0\" y1=\"14\" x2=\"120\" y2=\"14\" />\n",
" <line x1=\"0\" y1=\"15\" x2=\"120\" y2=\"15\" />\n",
" <line x1=\"0\" y1=\"17\" x2=\"120\" y2=\"17\" />\n",
" <line x1=\"0\" y1=\"18\" x2=\"120\" y2=\"18\" />\n",
" <line x1=\"0\" y1=\"19\" x2=\"120\" y2=\"19\" />\n",
" <line x1=\"0\" y1=\"21\" x2=\"120\" y2=\"21\" />\n",
" <line x1=\"0\" y1=\"22\" x2=\"120\" y2=\"22\" />\n",
" <line x1=\"0\" y1=\"24\" x2=\"120\" y2=\"24\" />\n",
" <line x1=\"0\" y1=\"25\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"25\" style=\"stroke-width:2\" />\n",
" <line x1=\"5\" y1=\"0\" x2=\"5\" y2=\"25\" />\n",
" <line x1=\"11\" y1=\"0\" x2=\"11\" y2=\"25\" />\n",
" <line x1=\"18\" y1=\"0\" x2=\"18\" y2=\"25\" />\n",
" <line x1=\"24\" y1=\"0\" x2=\"24\" y2=\"25\" />\n",
" <line x1=\"31\" y1=\"0\" x2=\"31\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"25\" />\n",
" <line x1=\"42\" y1=\"0\" x2=\"42\" y2=\"25\" />\n",
" <line x1=\"49\" y1=\"0\" x2=\"49\" y2=\"25\" />\n",
" <line x1=\"55\" y1=\"0\" x2=\"55\" y2=\"25\" />\n",
" <line x1=\"62\" y1=\"0\" x2=\"62\" y2=\"25\" />\n",
" <line x1=\"68\" y1=\"0\" x2=\"68\" y2=\"25\" />\n",
" <line x1=\"75\" y1=\"0\" x2=\"75\" y2=\"25\" />\n",
" <line x1=\"81\" y1=\"0\" x2=\"81\" y2=\"25\" />\n",
" <line x1=\"86\" y1=\"0\" x2=\"86\" y2=\"25\" />\n",
" <line x1=\"94\" y1=\"0\" x2=\"94\" y2=\"25\" />\n",
" <line x1=\"99\" y1=\"0\" x2=\"99\" y2=\"25\" />\n",
" <line x1=\"107\" y1=\"0\" x2=\"107\" y2=\"25\" />\n",
" <line x1=\"112\" y1=\"0\" x2=\"112\" y2=\"25\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.0,0.0 120.0,0.0 120.0,25.41261651458249 0.0,25.41261651458249\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"45.412617\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >9228245</text>\n",
" <text x=\"140.000000\" y=\"12.706308\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,12.706308)\">720</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></div></li><li class='xr-section-item'><input id='section-c540654b-69c5-4dc1-8666-9b626d5c4052' class='xr-section-summary-in' type='checkbox' checked><label for='section-c540654b-69c5-4dc1-8666-9b626d5c4052' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2031-08-03T02:10:00 ... 2031-08-...</div><input id='attrs-fa6217f4-df18-4e3f-83a5-eb2acfe10876' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-fa6217f4-df18-4e3f-83a5-eb2acfe10876' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-46d2d64c-22e7-441b-a4b6-8d1d3c3d0ed1' class='xr-var-data-in' type='checkbox'><label for='data-46d2d64c-22e7-441b-a4b6-8d1d3c3d0ed1' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>base_date :</span></dt><dd>2031-07-29 02:00:00</dd><dt><span>long_name :</span></dt><dd>model time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array([&#x27;2031-08-03T02:10:00.000000000&#x27;, &#x27;2031-08-03T02:20:00.000000000&#x27;,\n",
" &#x27;2031-08-03T02:30:00.000000000&#x27;, ..., &#x27;2031-08-08T01:40:00.000000000&#x27;,\n",
" &#x27;2031-08-08T01:50:00.000000000&#x27;, &#x27;2031-08-08T02:00:00.000000000&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>x</span></div><div class='xr-var-dims'>(node)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(141973,), meta=np.ndarray&gt;</div><input id='attrs-4f056508-e831-4bf8-b363-5581c316fb6e' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-4f056508-e831-4bf8-b363-5581c316fb6e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-3b56202b-992f-48a6-b4f7-d49df2a26c29' class='xr-var-data-in' type='checkbox'><label for='data-3b56202b-992f-48a6-b4f7-d49df2a26c29' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>longitude</dd><dt><span>positive :</span></dt><dd>east</dd><dt><span>standard_name :</span></dt><dd>longitude</dd><dt><span>units :</span></dt><dd>degrees_east</dd></dl></div><div class='xr-var-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 70.41 MiB </td>\n",
" <td> 1.08 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (9228245,) </td>\n",
" <td> (141973,) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 65 chunks in 2 graph layers </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"170\" height=\"75\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"25\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"25\" style=\"stroke-width:2\" />\n",
" <line x1=\"5\" y1=\"0\" x2=\"5\" y2=\"25\" />\n",
" <line x1=\"11\" y1=\"0\" x2=\"11\" y2=\"25\" />\n",
" <line x1=\"18\" y1=\"0\" x2=\"18\" y2=\"25\" />\n",
" <line x1=\"24\" y1=\"0\" x2=\"24\" y2=\"25\" />\n",
" <line x1=\"31\" y1=\"0\" x2=\"31\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"25\" />\n",
" <line x1=\"42\" y1=\"0\" x2=\"42\" y2=\"25\" />\n",
" <line x1=\"49\" y1=\"0\" x2=\"49\" y2=\"25\" />\n",
" <line x1=\"55\" y1=\"0\" x2=\"55\" y2=\"25\" />\n",
" <line x1=\"62\" y1=\"0\" x2=\"62\" y2=\"25\" />\n",
" <line x1=\"68\" y1=\"0\" x2=\"68\" y2=\"25\" />\n",
" <line x1=\"75\" y1=\"0\" x2=\"75\" y2=\"25\" />\n",
" <line x1=\"81\" y1=\"0\" x2=\"81\" y2=\"25\" />\n",
" <line x1=\"86\" y1=\"0\" x2=\"86\" y2=\"25\" />\n",
" <line x1=\"94\" y1=\"0\" x2=\"94\" y2=\"25\" />\n",
" <line x1=\"99\" y1=\"0\" x2=\"99\" y2=\"25\" />\n",
" <line x1=\"107\" y1=\"0\" x2=\"107\" y2=\"25\" />\n",
" <line x1=\"112\" y1=\"0\" x2=\"112\" y2=\"25\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.0,0.0 120.0,0.0 120.0,25.412616514582485 0.0,25.412616514582485\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"45.412617\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >9228245</text>\n",
" <text x=\"140.000000\" y=\"12.706308\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,140.000000,12.706308)\">1</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></li><li class='xr-var-item'><div class='xr-var-name'><span>y</span></div><div class='xr-var-dims'>(node)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(141973,), meta=np.ndarray&gt;</div><input id='attrs-bb0d2a89-d4de-41a1-95da-454b82b10ea9' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-bb0d2a89-d4de-41a1-95da-454b82b10ea9' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-9b10d34f-8215-4972-9f31-22454080c2f7' class='xr-var-data-in' type='checkbox'><label for='data-9b10d34f-8215-4972-9f31-22454080c2f7' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>long_name :</span></dt><dd>latitude</dd><dt><span>positive :</span></dt><dd>north</dd><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>units :</span></dt><dd>degrees_north</dd></dl></div><div class='xr-var-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 70.41 MiB </td>\n",
" <td> 1.08 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (9228245,) </td>\n",
" <td> (141973,) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 65 chunks in 2 graph layers </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"170\" height=\"75\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"25\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"25\" style=\"stroke-width:2\" />\n",
" <line x1=\"5\" y1=\"0\" x2=\"5\" y2=\"25\" />\n",
" <line x1=\"11\" y1=\"0\" x2=\"11\" y2=\"25\" />\n",
" <line x1=\"18\" y1=\"0\" x2=\"18\" y2=\"25\" />\n",
" <line x1=\"24\" y1=\"0\" x2=\"24\" y2=\"25\" />\n",
" <line x1=\"31\" y1=\"0\" x2=\"31\" y2=\"25\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"25\" />\n",
" <line x1=\"42\" y1=\"0\" x2=\"42\" y2=\"25\" />\n",
" <line x1=\"49\" y1=\"0\" x2=\"49\" y2=\"25\" />\n",
" <line x1=\"55\" y1=\"0\" x2=\"55\" y2=\"25\" />\n",
" <line x1=\"62\" y1=\"0\" x2=\"62\" y2=\"25\" />\n",
" <line x1=\"68\" y1=\"0\" x2=\"68\" y2=\"25\" />\n",
" <line x1=\"75\" y1=\"0\" x2=\"75\" y2=\"25\" />\n",
" <line x1=\"81\" y1=\"0\" x2=\"81\" y2=\"25\" />\n",
" <line x1=\"86\" y1=\"0\" x2=\"86\" y2=\"25\" />\n",
" <line x1=\"94\" y1=\"0\" x2=\"94\" y2=\"25\" />\n",
" <line x1=\"99\" y1=\"0\" x2=\"99\" y2=\"25\" />\n",
" <line x1=\"107\" y1=\"0\" x2=\"107\" y2=\"25\" />\n",
" <line x1=\"112\" y1=\"0\" x2=\"112\" y2=\"25\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"25\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.0,0.0 120.0,0.0 120.0,25.412616514582485 0.0,25.412616514582485\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"45.412617\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >9228245</text>\n",
" <text x=\"140.000000\" y=\"12.706308\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(0,140.000000,12.706308)\">1</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-c91cf098-b7cb-46fa-a476-4ba0af1a1e35' class='xr-section-summary-in' type='checkbox' ><label for='section-c91cf098-b7cb-46fa-a476-4ba0af1a1e35' class='xr-section-summary' >Indexes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>time</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-f1c1ab99-b582-4eb4-b4d8-23b65ab68d85' class='xr-index-data-in' type='checkbox'/><label for='index-f1c1ab99-b582-4eb4-b4d8-23b65ab68d85' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(DatetimeIndex([&#x27;2031-08-03 02:10:00&#x27;, &#x27;2031-08-03 02:20:00&#x27;,\n",
" &#x27;2031-08-03 02:30:00&#x27;, &#x27;2031-08-03 02:40:00&#x27;,\n",
" &#x27;2031-08-03 02:50:00&#x27;, &#x27;2031-08-03 03:00:00&#x27;,\n",
" &#x27;2031-08-03 03:10:00&#x27;, &#x27;2031-08-03 03:20:00&#x27;,\n",
" &#x27;2031-08-03 03:30:00&#x27;, &#x27;2031-08-03 03:40:00&#x27;,\n",
" ...\n",
" &#x27;2031-08-08 00:30:00&#x27;, &#x27;2031-08-08 00:40:00&#x27;,\n",
" &#x27;2031-08-08 00:50:00&#x27;, &#x27;2031-08-08 01:00:00&#x27;,\n",
" &#x27;2031-08-08 01:10:00&#x27;, &#x27;2031-08-08 01:20:00&#x27;,\n",
" &#x27;2031-08-08 01:30:00&#x27;, &#x27;2031-08-08 01:40:00&#x27;,\n",
" &#x27;2031-08-08 01:50:00&#x27;, &#x27;2031-08-08 02:00:00&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;, name=&#x27;time&#x27;, length=720, freq=None))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-15d678c2-9844-4aeb-b49a-620557c74559' class='xr-section-summary-in' type='checkbox' checked><label for='section-15d678c2-9844-4aeb-b49a-620557c74559' class='xr-section-summary' >Attributes: <span>(5)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>location :</span></dt><dd>node</dd><dt><span>long_name :</span></dt><dd>water surface elevation above geoid</dd><dt><span>mesh :</span></dt><dd>adcirc_mesh</dd><dt><span>standard_name :</span></dt><dd>sea_surface_height_above_geoid</dd><dt><span>units :</span></dt><dd>m</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'zeta' (time: 720, node: 9228245)>\n",
"dask.array<open_dataset-07e4e31a5b972cd686bab63d3a63abe9zeta, shape=(720, 9228245), dtype=float64, chunksize=(10, 141973), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 2031-08-03T02:10:00 ... 2031-08-08T02:00:00\n",
" x (node) float64 dask.array<chunksize=(141973,), meta=np.ndarray>\n",
" y (node) float64 dask.array<chunksize=(141973,), meta=np.ndarray>\n",
"Dimensions without coordinates: node\n",
"Attributes:\n",
" location: node\n",
" long_name: water surface elevation above geoid\n",
" mesh: adcirc_mesh\n",
" standard_name: sea_surface_height_above_geoid\n",
" units: m"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds.zeta"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "users-users-pangeo",
"language": "python",
"name": "conda-env-users-users-pangeo-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"state": {},
"version_major": 2,
"version_minor": 0
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment