Skip to content

Instantly share code, notes, and snippets.

@jbusecke

jbusecke/testing_even_division_algo.ipynb

Created July 16, 2023 20:37
Show Gist options
  • Save jbusecke/fb546fbfd51fb764bc7ebd696b810df6 to your computer and use it in GitHub Desktop.
Save jbusecke/fb546fbfd51fb764bc7ebd696b810df6 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
testing_even_division_algo.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Useage examples for dynamic chunking\n",
"\n",
"Make up a test dataset"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array&lt;chunksize=(255, 255, 255), meta=np.ndarray&gt;</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-9a3694df-7784-4e52-b831-4c2a0f84d6e2' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-9a3694df-7784-4e52-b831-4c2a0f84d6e2' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span>x</span>: 1000</li><li><span>y</span>: 1000</li><li><span>time</span>: 1000</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-b9a170da-2536-4c9a-8996-24bac33761ba' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-b9a170da-2536-4c9a-8996-24bac33761ba' class='xr-section-summary' title='Expand/collapse section'>Coordinates: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-ec4453d0-d6f2-4fd8-a606-abeb8420924f' class='xr-section-summary-in' type='checkbox' checked><label for='section-ec4453d0-d6f2-4fd8-a606-abeb8420924f' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>data</span></div><div class='xr-var-dims'>(x, y, time)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(255, 255, 255), meta=np.ndarray&gt;</div><input id='attrs-165636db-247d-41ce-8a92-3882710707ed' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-165636db-247d-41ce-8a92-3882710707ed' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-0e30ef6b-489c-43e7-a411-1d9c12ebbe64' class='xr-var-data-in' type='checkbox'><label for='data-0e30ef6b-489c-43e7-a411-1d9c12ebbe64' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 7.45 GiB </td>\n",
" <td> 126.51 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (1000, 1000, 1000) </td>\n",
" <td> (255, 255, 255) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 64 chunks in 1 graph layer </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"250\" height=\"240\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"30\" x2=\"80\" y2=\"101\" />\n",
" <line x1=\"10\" y1=\"61\" x2=\"80\" y2=\"131\" />\n",
" <line x1=\"10\" y1=\"91\" x2=\"80\" y2=\"162\" />\n",
" <line x1=\"10\" y1=\"120\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"28\" y2=\"138\" />\n",
" <line x1=\"46\" y1=\"36\" x2=\"46\" y2=\"156\" />\n",
" <line x1=\"64\" y1=\"54\" x2=\"64\" y2=\"174\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 80.58823529411765,70.58823529411765 80.58823529411765,190.58823529411765 10.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"130\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"28\" y1=\"18\" x2=\"148\" y2=\"18\" />\n",
" <line x1=\"46\" y1=\"36\" x2=\"166\" y2=\"36\" />\n",
" <line x1=\"64\" y1=\"54\" x2=\"184\" y2=\"54\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"40\" y1=\"0\" x2=\"111\" y2=\"70\" />\n",
" <line x1=\"71\" y1=\"0\" x2=\"141\" y2=\"70\" />\n",
" <line x1=\"101\" y1=\"0\" x2=\"172\" y2=\"70\" />\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 130.0,0.0 200.58823529411765,70.58823529411765 80.58823529411765,70.58823529411765\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"80\" y1=\"101\" x2=\"200\" y2=\"101\" />\n",
" <line x1=\"80\" y1=\"131\" x2=\"200\" y2=\"131\" />\n",
" <line x1=\"80\" y1=\"162\" x2=\"200\" y2=\"162\" />\n",
" <line x1=\"80\" y1=\"190\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
" <line x1=\"111\" y1=\"70\" x2=\"111\" y2=\"190\" />\n",
" <line x1=\"141\" y1=\"70\" x2=\"141\" y2=\"190\" />\n",
" <line x1=\"172\" y1=\"70\" x2=\"172\" y2=\"190\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"80.58823529411765,70.58823529411765 200.58823529411765,70.58823529411765 200.58823529411765,190.58823529411765 80.58823529411765,190.58823529411765\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"140.588235\" y=\"210.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >1000</text>\n",
" <text x=\"220.588235\" y=\"130.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,220.588235,130.588235)\">1000</text>\n",
" <text x=\"35.294118\" y=\"175.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,35.294118,175.294118)\">1000</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-207a2300-757c-4183-b193-8abc2d7d5ce1' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-207a2300-757c-4183-b193-8abc2d7d5ce1' class='xr-section-summary' title='Expand/collapse section'>Indexes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-ed809444-a650-4086-a1e0-a1496c57e99d' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-ed809444-a650-4086-a1e0-a1496c57e99d' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array<chunksize=(255, 255, 255), meta=np.ndarray>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import dask.array as dsa\n",
"import xarray as xr\n",
"from dynamic_target_chunks import dynamic_target_chunks_from_schema\n",
"from aggregation import dataset_to_schema\n",
"\n",
"ds = xr.DataArray(dsa.random.random([1000, 1000, 1000]), dims=['x', 'y','time']).to_dataset(name='data')\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now lets simulate Rich's case. We want have 10MB chunks and end up with 10 times more chunks along one of the dimensions (e.g. 'time')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array&lt;chunksize=(200, 250, 25), meta=np.ndarray&gt;</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-561f8c2a-6406-46bf-a977-c62fd2988a04' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-561f8c2a-6406-46bf-a977-c62fd2988a04' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span>x</span>: 1000</li><li><span>y</span>: 1000</li><li><span>time</span>: 1000</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-b3df4671-003f-4c5a-a04f-7a69b409bad3' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-b3df4671-003f-4c5a-a04f-7a69b409bad3' class='xr-section-summary' title='Expand/collapse section'>Coordinates: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-5c909480-a666-4d41-8450-13aea94743e4' class='xr-section-summary-in' type='checkbox' checked><label for='section-5c909480-a666-4d41-8450-13aea94743e4' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>data</span></div><div class='xr-var-dims'>(x, y, time)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(200, 250, 25), meta=np.ndarray&gt;</div><input id='attrs-4ead5fa0-1d3f-4666-98af-4d5d64958b0a' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-4ead5fa0-1d3f-4666-98af-4d5d64958b0a' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-93c558ff-b120-4484-8584-df27b37d56ff' class='xr-var-data-in' type='checkbox'><label for='data-93c558ff-b120-4484-8584-df27b37d56ff' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 7.45 GiB </td>\n",
" <td> 9.54 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (1000, 1000, 1000) </td>\n",
" <td> (200, 250, 25) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 800 chunks in 2 graph layers </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"250\" height=\"240\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"30\" x2=\"80\" y2=\"100\" />\n",
" <line x1=\"10\" y1=\"60\" x2=\"80\" y2=\"130\" />\n",
" <line x1=\"10\" y1=\"90\" x2=\"80\" y2=\"160\" />\n",
" <line x1=\"10\" y1=\"120\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"24\" y2=\"134\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"148\" />\n",
" <line x1=\"52\" y1=\"42\" x2=\"52\" y2=\"162\" />\n",
" <line x1=\"66\" y1=\"56\" x2=\"66\" y2=\"176\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 80.58823529411765,70.58823529411765 80.58823529411765,190.58823529411765 10.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"130\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"24\" y1=\"14\" x2=\"144\" y2=\"14\" />\n",
" <line x1=\"38\" y1=\"28\" x2=\"158\" y2=\"28\" />\n",
" <line x1=\"52\" y1=\"42\" x2=\"172\" y2=\"42\" />\n",
" <line x1=\"66\" y1=\"56\" x2=\"186\" y2=\"56\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"16\" y1=\"0\" x2=\"86\" y2=\"70\" />\n",
" <line x1=\"22\" y1=\"0\" x2=\"92\" y2=\"70\" />\n",
" <line x1=\"28\" y1=\"0\" x2=\"98\" y2=\"70\" />\n",
" <line x1=\"34\" y1=\"0\" x2=\"104\" y2=\"70\" />\n",
" <line x1=\"40\" y1=\"0\" x2=\"110\" y2=\"70\" />\n",
" <line x1=\"46\" y1=\"0\" x2=\"116\" y2=\"70\" />\n",
" <line x1=\"52\" y1=\"0\" x2=\"122\" y2=\"70\" />\n",
" <line x1=\"58\" y1=\"0\" x2=\"128\" y2=\"70\" />\n",
" <line x1=\"64\" y1=\"0\" x2=\"134\" y2=\"70\" />\n",
" <line x1=\"73\" y1=\"0\" x2=\"143\" y2=\"70\" />\n",
" <line x1=\"79\" y1=\"0\" x2=\"149\" y2=\"70\" />\n",
" <line x1=\"85\" y1=\"0\" x2=\"155\" y2=\"70\" />\n",
" <line x1=\"91\" y1=\"0\" x2=\"161\" y2=\"70\" />\n",
" <line x1=\"97\" y1=\"0\" x2=\"167\" y2=\"70\" />\n",
" <line x1=\"103\" y1=\"0\" x2=\"173\" y2=\"70\" />\n",
" <line x1=\"109\" y1=\"0\" x2=\"179\" y2=\"70\" />\n",
" <line x1=\"115\" y1=\"0\" x2=\"185\" y2=\"70\" />\n",
" <line x1=\"121\" y1=\"0\" x2=\"191\" y2=\"70\" />\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 130.0,0.0 200.58823529411765,70.58823529411765 80.58823529411765,70.58823529411765\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"80\" y1=\"100\" x2=\"200\" y2=\"100\" />\n",
" <line x1=\"80\" y1=\"130\" x2=\"200\" y2=\"130\" />\n",
" <line x1=\"80\" y1=\"160\" x2=\"200\" y2=\"160\" />\n",
" <line x1=\"80\" y1=\"190\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
" <line x1=\"86\" y1=\"70\" x2=\"86\" y2=\"190\" />\n",
" <line x1=\"92\" y1=\"70\" x2=\"92\" y2=\"190\" />\n",
" <line x1=\"98\" y1=\"70\" x2=\"98\" y2=\"190\" />\n",
" <line x1=\"104\" y1=\"70\" x2=\"104\" y2=\"190\" />\n",
" <line x1=\"110\" y1=\"70\" x2=\"110\" y2=\"190\" />\n",
" <line x1=\"116\" y1=\"70\" x2=\"116\" y2=\"190\" />\n",
" <line x1=\"122\" y1=\"70\" x2=\"122\" y2=\"190\" />\n",
" <line x1=\"128\" y1=\"70\" x2=\"128\" y2=\"190\" />\n",
" <line x1=\"134\" y1=\"70\" x2=\"134\" y2=\"190\" />\n",
" <line x1=\"143\" y1=\"70\" x2=\"143\" y2=\"190\" />\n",
" <line x1=\"149\" y1=\"70\" x2=\"149\" y2=\"190\" />\n",
" <line x1=\"155\" y1=\"70\" x2=\"155\" y2=\"190\" />\n",
" <line x1=\"161\" y1=\"70\" x2=\"161\" y2=\"190\" />\n",
" <line x1=\"167\" y1=\"70\" x2=\"167\" y2=\"190\" />\n",
" <line x1=\"173\" y1=\"70\" x2=\"173\" y2=\"190\" />\n",
" <line x1=\"179\" y1=\"70\" x2=\"179\" y2=\"190\" />\n",
" <line x1=\"185\" y1=\"70\" x2=\"185\" y2=\"190\" />\n",
" <line x1=\"191\" y1=\"70\" x2=\"191\" y2=\"190\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"80.58823529411765,70.58823529411765 200.58823529411765,70.58823529411765 200.58823529411765,190.58823529411765 80.58823529411765,190.58823529411765\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"140.588235\" y=\"210.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >1000</text>\n",
" <text x=\"220.588235\" y=\"130.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,220.588235,130.588235)\">1000</text>\n",
" <text x=\"35.294118\" y=\"175.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,35.294118,175.294118)\">1000</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-13034abc-d69f-46f2-b435-85f23feac087' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-13034abc-d69f-46f2-b435-85f23feac087' class='xr-section-summary' title='Expand/collapse section'>Indexes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-5b48ed2e-d40f-4a25-9226-2b91d3e1fa33' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-5b48ed2e-d40f-4a25-9226-2b91d3e1fa33' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array<chunksize=(200, 250, 25), meta=np.ndarray>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"target_chunks = dynamic_target_chunks_from_schema(\n",
" dataset_to_schema(ds),\n",
" target_chunk_ratio={'x':1, 'y':1, 'time':10},\n",
" target_chunk_nbytes=10e6,\n",
")\n",
"new_ds = ds.chunk(target_chunks)\n",
"new_ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Cool. We got really close to the 10MB chunksize and we have about 10 times the amount of chunks along the time dimension.\n",
"\n",
"Lets consider a simpler case next. We want 20 MB chunks along the time dimension only, but not chunk along the other dimensions:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Could not find any chunk combinations satisfying the size constraint. Consider increasing tolerance",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m target_chunks \u001b[39m=\u001b[39m dynamic_target_chunks_from_schema(\n\u001b[1;32m 2\u001b[0m dataset_to_schema(ds),\n\u001b[1;32m 3\u001b[0m target_chunk_ratio\u001b[39m=\u001b[39;49m{\u001b[39m'\u001b[39;49m\u001b[39mx\u001b[39;49m\u001b[39m'\u001b[39;49m:\u001b[39m-\u001b[39;49m\u001b[39m1\u001b[39;49m, \u001b[39m'\u001b[39;49m\u001b[39my\u001b[39;49m\u001b[39m'\u001b[39;49m:\u001b[39m-\u001b[39;49m\u001b[39m1\u001b[39;49m, \u001b[39m'\u001b[39;49m\u001b[39mtime\u001b[39;49m\u001b[39m'\u001b[39;49m:\u001b[39m10\u001b[39;49m},\n\u001b[1;32m 4\u001b[0m target_chunk_nbytes\u001b[39m=\u001b[39;49m\u001b[39m20e6\u001b[39;49m,\n\u001b[1;32m 5\u001b[0m )\n\u001b[1;32m 6\u001b[0m new_ds \u001b[39m=\u001b[39m ds\u001b[39m.\u001b[39mchunk(target_chunks)\n\u001b[1;32m 7\u001b[0m new_ds\n",
"File \u001b[0;32m~/Code/pangeo-forge-recipes/pangeo_forge_recipes/dynamic_target_chunks.py:75\u001b[0m, in \u001b[0;36mdynamic_target_chunks_from_schema\u001b[0;34m(schema, target_chunk_nbytes, target_chunk_ratio, nbytes_tolerance)\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[39m# If there are no matches in the range, the user has to increase the tolerance for this to work.\u001b[39;00m\n\u001b[1;32m 74\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(combinations_filtered) \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m---> 75\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 76\u001b[0m \u001b[39m'\u001b[39m\u001b[39mCould not find any chunk combinations satisfying the size constraint. Consider increasing tolerance\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 77\u001b[0m )\n\u001b[1;32m 79\u001b[0m \u001b[39m# Now that we have cominations in the memory size range we want, we can check which is closest to our \u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m# desired chunk ratio. We can think of this as comparing the angle of two vectors. \u001b[39;00m\n\u001b[1;32m 81\u001b[0m \u001b[39m# To compare them we need to normalize (we dont care about the amplitude here)\u001b[39;00m\n\u001b[1;32m 82\u001b[0m \n\u001b[1;32m 83\u001b[0m \u001b[39m# convert the combinations into the normalized inverse\u001b[39;00m\n\u001b[1;32m 84\u001b[0m ratio_combinations \u001b[39m=\u001b[39m [normalize(\u001b[39m1\u001b[39m\u001b[39m/\u001b[39mnp\u001b[39m.\u001b[39marray(c)) \u001b[39mfor\u001b[39;00m c \u001b[39min\u001b[39;00m combinations_filtered]\n",
"\u001b[0;31mValueError\u001b[0m: Could not find any chunk combinations satisfying the size constraint. Consider increasing tolerance"
]
}
],
"source": [
"target_chunks = dynamic_target_chunks_from_schema(\n",
" dataset_to_schema(ds),\n",
" target_chunk_ratio={'x':-1, 'y':-1, 'time':10},\n",
" target_chunk_nbytes=20e6,\n",
")\n",
"new_ds = ds.chunk(target_chunks)\n",
"new_ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Ooops. So that didnt work. But we can increase our tolerance for the size of the chunks"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array&lt;chunksize=(1000, 1000, 2), meta=np.ndarray&gt;</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-e8b124f9-ed17-4131-bf9a-30c757550e3c' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-e8b124f9-ed17-4131-bf9a-30c757550e3c' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span>x</span>: 1000</li><li><span>y</span>: 1000</li><li><span>time</span>: 1000</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-e82a1605-5dc6-4e04-80e3-8e1b205cbb11' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-e82a1605-5dc6-4e04-80e3-8e1b205cbb11' class='xr-section-summary' title='Expand/collapse section'>Coordinates: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-0909992e-13cf-461f-abc3-453cecff8a93' class='xr-section-summary-in' type='checkbox' checked><label for='section-0909992e-13cf-461f-abc3-453cecff8a93' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>data</span></div><div class='xr-var-dims'>(x, y, time)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(1000, 1000, 2), meta=np.ndarray&gt;</div><input id='attrs-93dfefbd-173f-4bef-9e52-577a3a8e7e6e' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-93dfefbd-173f-4bef-9e52-577a3a8e7e6e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-c438ead6-c9c7-49a5-9df9-658fbc192710' class='xr-var-data-in' type='checkbox'><label for='data-c438ead6-c9c7-49a5-9df9-658fbc192710' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
" <tr>\n",
" <td>\n",
" <table style=\"border-collapse: collapse;\">\n",
" <thead>\n",
" <tr>\n",
" <td> </td>\n",
" <th> Array </th>\n",
" <th> Chunk </th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" \n",
" <tr>\n",
" <th> Bytes </th>\n",
" <td> 7.45 GiB </td>\n",
" <td> 15.26 MiB </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <th> Shape </th>\n",
" <td> (1000, 1000, 1000) </td>\n",
" <td> (1000, 1000, 2) </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Dask graph </th>\n",
" <td colspan=\"2\"> 500 chunks in 4 graph layers </td>\n",
" </tr>\n",
" <tr>\n",
" <th> Data type </th>\n",
" <td colspan=\"2\"> float64 numpy.ndarray </td>\n",
" </tr>\n",
" </tbody>\n",
" </table>\n",
" </td>\n",
" <td>\n",
" <svg width=\"250\" height=\"240\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"10\" y1=\"120\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 80.58823529411765,70.58823529411765 80.58823529411765,190.58823529411765 10.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"130\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"16\" y1=\"0\" x2=\"86\" y2=\"70\" />\n",
" <line x1=\"22\" y1=\"0\" x2=\"93\" y2=\"70\" />\n",
" <line x1=\"28\" y1=\"0\" x2=\"99\" y2=\"70\" />\n",
" <line x1=\"35\" y1=\"0\" x2=\"105\" y2=\"70\" />\n",
" <line x1=\"41\" y1=\"0\" x2=\"112\" y2=\"70\" />\n",
" <line x1=\"47\" y1=\"0\" x2=\"118\" y2=\"70\" />\n",
" <line x1=\"54\" y1=\"0\" x2=\"124\" y2=\"70\" />\n",
" <line x1=\"60\" y1=\"0\" x2=\"130\" y2=\"70\" />\n",
" <line x1=\"66\" y1=\"0\" x2=\"137\" y2=\"70\" />\n",
" <line x1=\"73\" y1=\"0\" x2=\"143\" y2=\"70\" />\n",
" <line x1=\"79\" y1=\"0\" x2=\"149\" y2=\"70\" />\n",
" <line x1=\"85\" y1=\"0\" x2=\"156\" y2=\"70\" />\n",
" <line x1=\"92\" y1=\"0\" x2=\"162\" y2=\"70\" />\n",
" <line x1=\"98\" y1=\"0\" x2=\"168\" y2=\"70\" />\n",
" <line x1=\"104\" y1=\"0\" x2=\"175\" y2=\"70\" />\n",
" <line x1=\"111\" y1=\"0\" x2=\"181\" y2=\"70\" />\n",
" <line x1=\"117\" y1=\"0\" x2=\"187\" y2=\"70\" />\n",
" <line x1=\"123\" y1=\"0\" x2=\"194\" y2=\"70\" />\n",
" <line x1=\"130\" y1=\"0\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"10.0,0.0 130.0,0.0 200.58823529411765,70.58823529411765 80.58823529411765,70.58823529411765\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"200\" y2=\"70\" style=\"stroke-width:2\" />\n",
" <line x1=\"80\" y1=\"190\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"190\" style=\"stroke-width:2\" />\n",
" <line x1=\"86\" y1=\"70\" x2=\"86\" y2=\"190\" />\n",
" <line x1=\"93\" y1=\"70\" x2=\"93\" y2=\"190\" />\n",
" <line x1=\"99\" y1=\"70\" x2=\"99\" y2=\"190\" />\n",
" <line x1=\"105\" y1=\"70\" x2=\"105\" y2=\"190\" />\n",
" <line x1=\"112\" y1=\"70\" x2=\"112\" y2=\"190\" />\n",
" <line x1=\"118\" y1=\"70\" x2=\"118\" y2=\"190\" />\n",
" <line x1=\"124\" y1=\"70\" x2=\"124\" y2=\"190\" />\n",
" <line x1=\"130\" y1=\"70\" x2=\"130\" y2=\"190\" />\n",
" <line x1=\"137\" y1=\"70\" x2=\"137\" y2=\"190\" />\n",
" <line x1=\"143\" y1=\"70\" x2=\"143\" y2=\"190\" />\n",
" <line x1=\"149\" y1=\"70\" x2=\"149\" y2=\"190\" />\n",
" <line x1=\"156\" y1=\"70\" x2=\"156\" y2=\"190\" />\n",
" <line x1=\"162\" y1=\"70\" x2=\"162\" y2=\"190\" />\n",
" <line x1=\"168\" y1=\"70\" x2=\"168\" y2=\"190\" />\n",
" <line x1=\"175\" y1=\"70\" x2=\"175\" y2=\"190\" />\n",
" <line x1=\"181\" y1=\"70\" x2=\"181\" y2=\"190\" />\n",
" <line x1=\"187\" y1=\"70\" x2=\"187\" y2=\"190\" />\n",
" <line x1=\"194\" y1=\"70\" x2=\"194\" y2=\"190\" />\n",
" <line x1=\"200\" y1=\"70\" x2=\"200\" y2=\"190\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"80.58823529411765,70.58823529411765 200.58823529411765,70.58823529411765 200.58823529411765,190.58823529411765 80.58823529411765,190.58823529411765\" style=\"fill:#8B4903A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"140.588235\" y=\"210.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >1000</text>\n",
" <text x=\"220.588235\" y=\"130.588235\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,220.588235,130.588235)\">1000</text>\n",
" <text x=\"35.294118\" y=\"175.294118\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,35.294118,175.294118)\">1000</text>\n",
"</svg>\n",
" </td>\n",
" </tr>\n",
"</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-fc1fff5a-75b6-4dc0-b583-10b98584eafc' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-fc1fff5a-75b6-4dc0-b583-10b98584eafc' class='xr-section-summary' title='Expand/collapse section'>Indexes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-d4645bed-b186-4a18-9a9a-05d96299406c' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-d4645bed-b186-4a18-9a9a-05d96299406c' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (x: 1000, y: 1000, time: 1000)\n",
"Dimensions without coordinates: x, y, time\n",
"Data variables:\n",
" data (x, y, time) float64 dask.array<chunksize=(1000, 1000, 2), meta=np.ndarray>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"target_chunks = dynamic_target_chunks_from_schema(\n",
" dataset_to_schema(ds),\n",
" target_chunk_ratio={'x':-1, 'y':-1, 'time':10},\n",
" target_chunk_nbytes=20e6,\n",
" nbytes_tolerance=0.3, #within +-30% of our target\n",
")\n",
"new_ds = ds.chunk(target_chunks)\n",
"new_ds"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "xarray_cloud_sandbox",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment