Skip to content

Instantly share code, notes, and snippets.

@TomNicholas

TomNicholas/from_array.ipynb

Created May 24, 2023 19:03
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save TomNicholas/1040ebb0c13d53b65bdf870dae55330d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save TomNicholas/1040ebb0c13d53b65bdf870dae55330d to your computer and use it in GitHub Desktop.
Download ZIP
Cubed graphs created when creating xarray objects
Display the source blob
Display the rendered blob
Raw
from_array.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "1a16e2aa-2dfd-483e-990e-0299292a83cf",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import xarray as xr\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3a4444bf-5024-423d-8857-0784dacbe0fd",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import cubed\n",
"import cubed.random"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "68970c8b-d1bf-4f0a-bee3-4f2e68a6f707",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"spec = cubed.Spec(work_dir=\"tmp\", max_mem=3_000_000_000)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "4a655ab2-b337-4d1f-ade3-678e92bae058",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;U&#x27; (time: 5, face: 1, j: 987, i: 1920)&gt;\n",
"cubed.Array&lt;array-003, shape=(5, 1, 987, 1920), dtype=float64, chunks=((5,), (1,), (987,), (1920,))&gt;\n",
"Dimensions without coordinates: time, face, j, i</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'U'</div><ul class='xr-dim-list'><li><span>time</span>: 5</li><li><span>face</span>: 1</li><li><span>j</span>: 987</li><li><span>i</span>: 1920</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-1cde8c39-9624-4e20-b06b-864a98784923' class='xr-array-in' type='checkbox' checked><label for='section-1cde8c39-9624-4e20-b06b-864a98784923' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>...</span></div><div class='xr-array-data'><pre>cubed.Array&lt;array-003, shape=(5, 1, 987, 1920), dtype=float64, chunks=((5,), (1,), (987,), (1920,))&gt;</pre></div></div></li><li class='xr-section-item'><input id='section-50e3ecf4-3466-4a2a-8996-77ec3de4a350' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-50e3ecf4-3466-4a2a-8996-77ec3de4a350' class='xr-section-summary' title='Expand/collapse section'>Coordinates: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-3cc6e907-c8e5-4bcb-bd5b-3ac4e2dcb2c6' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-3cc6e907-c8e5-4bcb-bd5b-3ac4e2dcb2c6' class='xr-section-summary' title='Expand/collapse section'>Indexes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'></ul></div></li><li class='xr-section-item'><input id='section-84931663-bf54-427c-b3b7-0d09ba2c6c94' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-84931663-bf54-427c-b3b7-0d09ba2c6c94' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'U' (time: 5, face: 1, j: 987, i: 1920)>\n",
"cubed.Array<array-003, shape=(5, 1, 987, 1920), dtype=float64, chunks=((5,), (1,), (987,), (1920,))>\n",
"Dimensions without coordinates: time, face, j, i"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"U = xr.DataArray(\n",
" name='U',\n",
" data=cubed.random.random((5, 1, 987, 1920), chunks=(10, 1, -1, -1), spec=spec),\n",
" dims=[\"time\", \"face\", \"j\", \"i\"],\n",
")\n",
"U"
]
},
{
"cell_type": "markdown",
"id": "aa37c884-a757-44a7-8efe-735620efab56",
"metadata": {},
"source": [
"(`ChunkManager.from_array` does not get called if `cubed.Array` supplied explicitly)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "4474959f-1418-48e2-b4d5-90b11915db74",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"U = xr.DataArray(\n",
" name='U',\n",
" data=np.random.random((5, 1, 987, 1920)),\n",
" dims=[\"time\", \"face\", \"j\", \"i\"],\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "81d0778d-bbea-47f4-ab87-aa7ad8725696",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CubedManager.from_array() called on type <class 'numpy.ndarray'>\n"
]
}
],
"source": [
"U_chunked = U.chunk(\n",
" chunks=(10, 1, -1, -1), \n",
" chunked_array_type='cubed',\n",
" from_array_kwargs=dict(spec=spec),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "0603b051-4c93-47db-9d0e-1022b2577079",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"image/svg+xml": [
"<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"196pt\" height=\"150pt\" viewBox=\"0.00 0.00 195.75 149.50\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 145.5)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-145.5 191.75,-145.5 191.75,4 -4,4\"/>\n",
"<text text-anchor=\"middle\" x=\"93.88\" y=\"-20\" font-family=\"Times,serif\" font-size=\"10.00\">num tasks: 1</text>\n",
"<text text-anchor=\"middle\" x=\"93.88\" y=\"-7.25\" font-family=\"Times,serif\" font-size=\"10.00\">max projected memory: 303.2 MB</text>\n",
"<!-- array&#45;004 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>array-004</title>\n",
"<g id=\"a_node1\"><a xlink:title=\"shape: (5, 1, 987, 1920)\n",
"chunks: (5, 1, 987, 1920)\n",
"dtype: float64\n",
"chunk memory: 75.8 MB\n",
"\n",
"calls: chunk -&gt; chunk -&gt; &lt;dictcomp&gt; -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; empty -&gt; full\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"84.88,-141.5 11.88,-141.5 11.88,-105.5 84.88,-105.5 84.88,-141.5\"/>\n",
"<text text-anchor=\"middle\" x=\"48.38\" y=\"-126.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-004</text>\n",
"<text text-anchor=\"middle\" x=\"48.38\" y=\"-114\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array </text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;006 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>array-006</title>\n",
"<g id=\"a_node3\"><a xlink:title=\"shape: (5, 1, 987, 1920)\n",
"chunks: (5, 1, 987, 1920)\n",
"dtype: float64\n",
"chunk memory: 75.8 MB\n",
"\n",
"projected memory: 303.2 MB\n",
"tasks: 1\n",
"calls: chunk -&gt; &lt;dictcomp&gt; -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; map_blocks -&gt; _map_blocks -&gt; blockwise\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"#dcbeff\" stroke=\"black\" points=\"140.75,-69.5 46,-69.5 46,-33.5 140.75,-33.5 140.75,-69.5\"/>\n",
"<text text-anchor=\"middle\" x=\"93.38\" y=\"-54.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-006</text>\n",
"<text text-anchor=\"middle\" x=\"93.38\" y=\"-42\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array (bw)</text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;004&#45;&gt;array&#45;006 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>array-004-&gt;array-006</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M59.5,-105.2C64.62,-97.23 70.81,-87.6 76.49,-78.76\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"79.92,-80.91 82.38,-70.6 74.03,-77.12 79.92,-80.91\"/>\n",
"</g>\n",
"<!-- array&#45;005 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>array-005</title>\n",
"<g id=\"a_node2\"><a xlink:title=\"shape: (1, 1, 1, 1)\n",
"chunks: (1, 1, 1, 1)\n",
"dtype: int32\n",
"chunk memory: 4 bytes\n",
"\n",
"calls: chunk -&gt; chunk -&gt; &lt;dictcomp&gt; -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; map_blocks -&gt; asarray\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"175.88,-141.5 102.88,-141.5 102.88,-105.5 175.88,-105.5 175.88,-141.5\"/>\n",
"<text text-anchor=\"middle\" x=\"139.38\" y=\"-126.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-005</text>\n",
"<text text-anchor=\"middle\" x=\"139.38\" y=\"-114\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array </text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;005&#45;&gt;array&#45;006 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>array-005-&gt;array-006</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M128,-105.2C122.77,-97.23 116.44,-87.6 110.63,-78.76\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"113.03,-77.04 104.61,-70.6 107.18,-80.88 113.03,-77.04\"/>\n",
"</g>\n",
"</g>\n",
"</svg>"
],
"text/plain": [
"<IPython.core.display.SVG object>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"U_chunked.data.visualize()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "93c0ca86-61d7-45f1-ba12-9a65e11e7e40",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.backends.zarr.ZarrStore at 0x7f8bed9a3e40>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"U.to_dataset().to_zarr('test.zarr')"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "4ebe5930-f562-47a1-9918-4b852c59ebd8",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CubedManager.from_array() called on type <class 'xarray.core.indexing.ImplicitToExplicitIndexingAdapter'>\n"
]
}
],
"source": [
"U_from_zarr = xr.open_dataset('test.zarr', engine='zarr', chunked_array_type='cubed', chunks={})"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "4e15e7a0-b548-45db-9a5b-9104239213c4",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"image/svg+xml": [
"<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"196pt\" height=\"222pt\" viewBox=\"0.00 0.00 195.75 221.50\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 217.5)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-217.5 191.75,-217.5 191.75,4 -4,4\"/>\n",
"<text text-anchor=\"middle\" x=\"93.88\" y=\"-20\" font-family=\"Times,serif\" font-size=\"10.00\">num tasks: 81</text>\n",
"<text text-anchor=\"middle\" x=\"93.88\" y=\"-7.25\" font-family=\"Times,serif\" font-size=\"10.00\">max projected memory: 200.0 MB</text>\n",
"<!-- array&#45;007 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>array-007</title>\n",
"<g id=\"a_node1\"><a xlink:title=\"shape: (5, 1, 987, 1920)\n",
"chunks: (1, 1, 247, 480)\n",
"dtype: float64\n",
"chunk memory: 948.5 KB\n",
"\n",
"calls: open_dataset -&gt; _dataset_from_backend_dataset -&gt; _chunk_ds -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; empty -&gt; full\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"76.75,-141.5 3.75,-141.5 3.75,-105.5 76.75,-105.5 76.75,-141.5\"/>\n",
"<text text-anchor=\"middle\" x=\"40.25\" y=\"-126.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-007</text>\n",
"<text text-anchor=\"middle\" x=\"40.25\" y=\"-114\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array </text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;010 -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>array-010</title>\n",
"<g id=\"a_node4\"><a xlink:title=\"shape: (5, 1, 987, 1920)\n",
"chunks: (1, 1, 247, 480)\n",
"dtype: float64\n",
"chunk memory: 948.5 KB\n",
"\n",
"projected memory: 103.8 MB\n",
"tasks: 80\n",
"calls: _dataset_from_backend_dataset -&gt; _chunk_ds -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; map_blocks -&gt; _map_blocks -&gt; blockwise\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"#dcbeff\" stroke=\"black\" points=\"136.62,-69.5 41.88,-69.5 41.88,-33.5 136.62,-33.5 136.62,-69.5\"/>\n",
"<text text-anchor=\"middle\" x=\"89.25\" y=\"-54.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-010</text>\n",
"<text text-anchor=\"middle\" x=\"89.25\" y=\"-42\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array (bw)</text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;007&#45;&gt;array&#45;010 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>array-007-&gt;array-010</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M52.36,-105.2C58,-97.14 64.82,-87.39 71.06,-78.48\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"74.41,-80.8 77.28,-70.6 68.67,-76.79 74.41,-80.8\"/>\n",
"</g>\n",
"<!-- array&#45;008 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>array-008</title>\n",
"<g id=\"a_node2\"><a xlink:title=\"shape: (5, 1, 4, 4)\n",
"chunks: (5, 1, 4, 4)\n",
"dtype: int32\n",
"chunk memory: 320 bytes\n",
"\n",
"calls: open_dataset -&gt; _dataset_from_backend_dataset -&gt; _chunk_ds -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; map_blocks -&gt; asarray\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"175.75,-213.5 102.75,-213.5 102.75,-177.5 175.75,-177.5 175.75,-213.5\"/>\n",
"<text text-anchor=\"middle\" x=\"139.25\" y=\"-198.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-008</text>\n",
"<text text-anchor=\"middle\" x=\"139.25\" y=\"-186\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array </text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;009 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>array-009</title>\n",
"<g id=\"a_node3\"><a xlink:title=\"shape: (5, 1, 4, 4)\n",
"chunks: (1, 1, 1, 1)\n",
"dtype: int32\n",
"chunk memory: 4 bytes\n",
"\n",
"projected memory: 200.0 MB\n",
"tasks: 1\n",
"calls: open_dataset -&gt; _dataset_from_backend_dataset -&gt; _chunk_ds -&gt; _maybe_chunk -&gt; chunk -&gt; from_array -&gt; from_array -&gt; map_direct -&gt; map_blocks -&gt; rechunk\n",
"line: 62 in from_array\">\n",
"<polygon fill=\"#aaffc3\" stroke=\"black\" points=\"184,-141.5 94.5,-141.5 94.5,-105.5 184,-105.5 184,-141.5\"/>\n",
"<text text-anchor=\"middle\" x=\"139.25\" y=\"-126.75\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">array-009</text>\n",
"<text text-anchor=\"middle\" x=\"139.25\" y=\"-114\" font-family=\"Helvetica,sans-Serif\" font-size=\"10.00\">from_array (rc)</text>\n",
"</a>\n",
"</g>\n",
"</g>\n",
"<!-- array&#45;008&#45;&gt;array&#45;009 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>array-008-&gt;array-009</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M139.25,-177.2C139.25,-169.74 139.25,-160.82 139.25,-152.47\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"142.75,-152.6 139.25,-142.6 135.75,-152.6 142.75,-152.6\"/>\n",
"</g>\n",
"<!-- array&#45;009&#45;&gt;array&#45;010 -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>array-009-&gt;array-010</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M126.89,-105.2C121.14,-97.14 114.17,-87.39 107.81,-78.48\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"110.13,-76.71 101.47,-70.6 104.43,-80.78 110.13,-76.71\"/>\n",
"</g>\n",
"</g>\n",
"</svg>"
],
"text/plain": [
"<IPython.core.display.SVG object>"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"U_from_zarr['U'].data.visualize()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6f915e69-4fee-42e8-aee4-2a2e77afe3b2",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:cubed_xarray] *",
"language": "python",
"name": "conda-env-cubed_xarray-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment