Skip to content

Instantly share code, notes, and snippets.

@eric-czech

eric-czech/pairwise_numba_options.ipynb

Last active October 6, 2020 13:46
Show Gist options
  • Save eric-czech/eb1913775b20456c4dc9eac6677dab9e to your computer and use it in GitHub Desktop.
Save eric-czech/eb1913775b20456c4dc9eac6677dab9e to your computer and use it in GitHub Desktop.
Download ZIP
pairwise_numba_options
Display the source blob
Display the rendered blob
Raw
pairwise_numba_options.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pairwise Numba Calculations\n",
"\n",
"This compares the following pairwise implementations:\n",
"\n",
"- 1D chunked pairwise block functions using njit and guvectorize\n",
" - Surprisingly, this is how things like [dask.array.matmul](https://github.com/dask/dask/blob/master/dask/array/routines.py#L313) actually work\n",
" - They use `da.blockwise` to pair up rows of blocks, which are concatenated in memory\n",
"- The guvectorize variants above are compared with vector inputs and array inputs (so the custom function is responsible for looping)\n",
"- Fully chunked pairwise function, like that mentioned in https://github.com/pystatgen/sgkit/issues/241#issuecomment-698878378"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div class=\"bk-root\">\n",
" <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
" <span id=\"1001\">Loading BokehJS ...</span>\n",
" </div>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"\n",
"(function(root) {\n",
" function now() {\n",
" return new Date();\n",
" }\n",
"\n",
" var force = true;\n",
"\n",
" if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
" root._bokeh_onload_callbacks = [];\n",
" root._bokeh_is_loading = undefined;\n",
" }\n",
"\n",
" var JS_MIME_TYPE = 'application/javascript';\n",
" var HTML_MIME_TYPE = 'text/html';\n",
" var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
" var CLASS_NAME = 'output_bokeh rendered_html';\n",
"\n",
" /**\n",
" * Render data to the DOM node\n",
" */\n",
" function render(props, node) {\n",
" var script = document.createElement(\"script\");\n",
" node.appendChild(script);\n",
" }\n",
"\n",
" /**\n",
" * Handle when an output is cleared or removed\n",
" */\n",
" function handleClearOutput(event, handle) {\n",
" var cell = handle.cell;\n",
"\n",
" var id = cell.output_area._bokeh_element_id;\n",
" var server_id = cell.output_area._bokeh_server_id;\n",
" // Clean up Bokeh references\n",
" if (id != null && id in Bokeh.index) {\n",
" Bokeh.index[id].model.document.clear();\n",
" delete Bokeh.index[id];\n",
" }\n",
"\n",
" if (server_id !== undefined) {\n",
" // Clean up Bokeh references\n",
" var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
" cell.notebook.kernel.execute(cmd, {\n",
" iopub: {\n",
" output: function(msg) {\n",
" var id = msg.content.text.trim();\n",
" if (id in Bokeh.index) {\n",
" Bokeh.index[id].model.document.clear();\n",
" delete Bokeh.index[id];\n",
" }\n",
" }\n",
" }\n",
" });\n",
" // Destroy server and session\n",
" var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
" cell.notebook.kernel.execute(cmd);\n",
" }\n",
" }\n",
"\n",
" /**\n",
" * Handle when a new output is added\n",
" */\n",
" function handleAddOutput(event, handle) {\n",
" var output_area = handle.output_area;\n",
" var output = handle.output;\n",
"\n",
" // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
" if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
" return\n",
" }\n",
"\n",
" var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
"\n",
" if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
" toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
" // store reference to embed id on output_area\n",
" output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
" }\n",
" if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
" var bk_div = document.createElement(\"div\");\n",
" bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
" var script_attrs = bk_div.children[0].attributes;\n",
" for (var i = 0; i < script_attrs.length; i++) {\n",
" toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
" toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
" }\n",
" // store reference to server id on output_area\n",
" output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
" }\n",
" }\n",
"\n",
" function register_renderer(events, OutputArea) {\n",
"\n",
" function append_mime(data, metadata, element) {\n",
" // create a DOM node to render to\n",
" var toinsert = this.create_output_subarea(\n",
" metadata,\n",
" CLASS_NAME,\n",
" EXEC_MIME_TYPE\n",
" );\n",
" this.keyboard_manager.register_events(toinsert);\n",
" // Render to node\n",
" var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
" render(props, toinsert[toinsert.length - 1]);\n",
" element.append(toinsert);\n",
" return toinsert\n",
" }\n",
"\n",
" /* Handle when an output is cleared or removed */\n",
" events.on('clear_output.CodeCell', handleClearOutput);\n",
" events.on('delete.Cell', handleClearOutput);\n",
"\n",
" /* Handle when a new output is added */\n",
" events.on('output_added.OutputArea', handleAddOutput);\n",
"\n",
" /**\n",
" * Register the mime type and append_mime function with output_area\n",
" */\n",
" OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
" /* Is output safe? */\n",
" safe: true,\n",
" /* Index of renderer in `output_area.display_order` */\n",
" index: 0\n",
" });\n",
" }\n",
"\n",
" // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
" if (root.Jupyter !== undefined) {\n",
" var events = require('base/js/events');\n",
" var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
"\n",
" if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
" register_renderer(events, OutputArea);\n",
" }\n",
" }\n",
"\n",
" \n",
" if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
" root._bokeh_timeout = Date.now() + 5000;\n",
" root._bokeh_failed_load = false;\n",
" }\n",
"\n",
" var NB_LOAD_WARNING = {'data': {'text/html':\n",
" \"<div style='background-color: #fdd'>\\n\"+\n",
" \"<p>\\n\"+\n",
" \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
" \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
" \"</p>\\n\"+\n",
" \"<ul>\\n\"+\n",
" \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
" \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
" \"</ul>\\n\"+\n",
" \"<code>\\n\"+\n",
" \"from bokeh.resources import INLINE\\n\"+\n",
" \"output_notebook(resources=INLINE)\\n\"+\n",
" \"</code>\\n\"+\n",
" \"</div>\"}};\n",
"\n",
" function display_loaded() {\n",
" var el = document.getElementById(\"1001\");\n",
" if (el != null) {\n",
" el.textContent = \"BokehJS is loading...\";\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" if (el != null) {\n",
" el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
" }\n",
" } else if (Date.now() < root._bokeh_timeout) {\n",
" setTimeout(display_loaded, 100)\n",
" }\n",
" }\n",
"\n",
"\n",
" function run_callbacks() {\n",
" try {\n",
" root._bokeh_onload_callbacks.forEach(function(callback) {\n",
" if (callback != null)\n",
" callback();\n",
" });\n",
" } finally {\n",
" delete root._bokeh_onload_callbacks\n",
" }\n",
" console.debug(\"Bokeh: all callbacks have finished\");\n",
" }\n",
"\n",
" function load_libs(css_urls, js_urls, callback) {\n",
" if (css_urls == null) css_urls = [];\n",
" if (js_urls == null) js_urls = [];\n",
"\n",
" root._bokeh_onload_callbacks.push(callback);\n",
" if (root._bokeh_is_loading > 0) {\n",
" console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
" return null;\n",
" }\n",
" if (js_urls == null || js_urls.length === 0) {\n",
" run_callbacks();\n",
" return null;\n",
" }\n",
" console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
" root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
"\n",
" function on_load() {\n",
" root._bokeh_is_loading--;\n",
" if (root._bokeh_is_loading === 0) {\n",
" console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
" run_callbacks()\n",
" }\n",
" }\n",
"\n",
" function on_error() {\n",
" console.error(\"failed to load \" + url);\n",
" }\n",
"\n",
" for (var i = 0; i < css_urls.length; i++) {\n",
" var url = css_urls[i];\n",
" const element = document.createElement(\"link\");\n",
" element.onload = on_load;\n",
" element.onerror = on_error;\n",
" element.rel = \"stylesheet\";\n",
" element.type = \"text/css\";\n",
" element.href = url;\n",
" console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
" document.body.appendChild(element);\n",
" }\n",
"\n",
" const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n",
"\n",
" for (var i = 0; i < js_urls.length; i++) {\n",
" var url = js_urls[i];\n",
" var element = document.createElement('script');\n",
" element.onload = on_load;\n",
" element.onerror = on_error;\n",
" element.async = false;\n",
" element.src = url;\n",
" if (url in hashes) {\n",
" element.crossOrigin = \"anonymous\";\n",
" element.integrity = \"sha384-\" + hashes[url];\n",
" }\n",
" console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
" document.head.appendChild(element);\n",
" }\n",
" };\n",
"\n",
" function inject_raw_css(css) {\n",
" const element = document.createElement(\"style\");\n",
" element.appendChild(document.createTextNode(css));\n",
" document.body.appendChild(element);\n",
" }\n",
"\n",
" \n",
" var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n",
" var css_urls = [];\n",
" \n",
"\n",
" var inline_js = [\n",
" function(Bokeh) {\n",
" Bokeh.set_log_level(\"info\");\n",
" },\n",
" function(Bokeh) {\n",
" \n",
" \n",
" }\n",
" ];\n",
"\n",
" function run_inline_js() {\n",
" \n",
" if (root.Bokeh !== undefined || force === true) {\n",
" \n",
" for (var i = 0; i < inline_js.length; i++) {\n",
" inline_js[i].call(root, root.Bokeh);\n",
" }\n",
" if (force === true) {\n",
" display_loaded();\n",
" }} else if (Date.now() < root._bokeh_timeout) {\n",
" setTimeout(run_inline_js, 100);\n",
" } else if (!root._bokeh_failed_load) {\n",
" console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
" root._bokeh_failed_load = true;\n",
" } else if (force !== true) {\n",
" var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
" cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
" }\n",
"\n",
" }\n",
"\n",
" if (root._bokeh_is_loading === 0) {\n",
" console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
" run_inline_js();\n",
" } else {\n",
" load_libs(css_urls, js_urls, function() {\n",
" console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
" run_inline_js();\n",
" });\n",
" }\n",
"}(window));"
],
"application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"<div style='background-color: #fdd'>\\n\"+\n \"<p>\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"</p>\\n\"+\n \"<ul>\\n\"+\n \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n \"<li>use INLINE resources instead, as so:</li>\\n\"+\n \"</ul>\\n\"+\n \"<code>\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"</code>\\n\"+\n \"</div>\"}};\n\n function display_loaded() {\n var el = document.getElementById(\"1001\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n if (url in hashes) {\n element.crossOrigin = \"anonymous\";\n element.integrity = \"sha384-\" + hashes[url];\n }\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import numpy as np\n",
"import dask.array as da\n",
"from numba import guvectorize, njit\n",
"from dask.diagnostics import ProgressBar, ResourceProfiler\n",
"from bokeh.io import output_notebook\n",
"output_notebook()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 6.00 MB </td> <td> 270.00 kB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 2000) </td> <td> (900, 300) </td></tr>\n",
" <tr><th> Count </th><td> 28 Tasks </td><td> 28 Chunks </td></tr>\n",
" <tr><th> Type </th><td> int8 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"130\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"80\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"80\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"80\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"80\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"80\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"12\" y1=\"0\" x2=\"12\" y2=\"120\" />\n",
" <line x1=\"24\" y1=\"0\" x2=\"24\" y2=\"120\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"48\" y1=\"0\" x2=\"48\" y2=\"120\" />\n",
" <line x1=\"60\" y1=\"0\" x2=\"60\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"80\" y1=\"0\" x2=\"80\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 80.000000,0.000000 80.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"40.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >2000</text>\n",
" <text x=\"100.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,100.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<randint, shape=(3000, 2000), dtype=int8, chunksize=(900, 300), chunktype=numpy.ndarray>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rs = da.random.RandomState(0)\n",
"# Pick a somewhat large array size with unequal chunk at end\n",
"x = rs.randint(0, 3, size=(3000, 2000), dtype='i1', chunks=(900, 300))\n",
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Baseline\n",
"\n",
"Compare to `da.corrcoef`"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 0.8s\n",
"CPU times: user 6.9 s, sys: 5.92 s, total: 12.8 s\n",
"Wall time: 1.45 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = da.corrcoef(x).compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 1\n",
"\n",
"Use `guvectorize` with array inputs and apply vectorized functions (i.e. avoid loops). \n",
"\n",
"This is fastest but difficult to use since many ufunc reduce functions are not implemented, i.e. you cannot even use something like `np.mean(x, axis=0)`."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 44 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<corrcoef, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@guvectorize(['void(int8[:,:], int8[:,:], float64[:,:])'], '(n,k),(m,k)->(n,m)', fastmath=True)\n",
"def corrcoef(x, y, out):\n",
" # Ufunc reduce not supported for pretty much all numpy functions except sum:\n",
" # https://numba.pydata.org/numba-doc/dev/reference/numpysupported.html\n",
" m = x.shape[1]\n",
" x, y = x.T, y.T\n",
" xc, yc = x - x.sum(0)/m, y - y.sum(0)/m\n",
" x = xc / np.sqrt((xc**2).sum(0)/m)\n",
" y = yc / np.sqrt((yc**2).sum(0)/m)\n",
" out[:,:] = np.dot(x.T, y) / m\n",
" \n",
"def pairwise(x, fn):\n",
" return da.blockwise(\n",
" fn, 'jk', \n",
" x, 'ji',\n",
" x, 'ki', \n",
" dtype='float64',\n",
" concatenate=True\n",
" )\n",
"\n",
"xc = pairwise(x, corrcoef)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 0.4s\n",
"CPU times: user 5 s, sys: 3.35 s, total: 8.35 s\n",
"Wall time: 1.44 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 2\n",
"\n",
"Use `guvectorize` with array inputs and more generalizable looping."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 72 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<corrcoef, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@guvectorize(['void(int8[:,:], int8[:,:], float64[:,:])'], '(n,k),(k,m)->(n,m)', fastmath=True)\n",
"def corrcoef(x, y, out):\n",
" # Ufunc reduce not supported for pretty much all numpy functions except sum:\n",
" # https://numba.pydata.org/numba-doc/dev/reference/numpysupported.html\n",
" n, m = x.shape[0], y.shape[1]\n",
" \n",
" def compute(xv, yv):\n",
" xvc = (xv - xv.mean()) / np.std(xv)\n",
" yvc = (yv - yv.mean()) / np.std(yv)\n",
" return (xvc * yvc).mean()\n",
"\n",
" for i in range(n):\n",
" for j in range(m):\n",
" out[i, j] = compute(x[i], y[:, j])\n",
" # Alternatively (but slower):\n",
" # out[i, j] = np.corrcoef(x[i], y[:, j])[0, 1]\n",
" \n",
"def pairwise(x, fn):\n",
" return da.blockwise(\n",
" fn, 'jk', \n",
" x, 'ji',\n",
" x.T, 'ik', \n",
" dtype='float64',\n",
" concatenate=True\n",
" )\n",
"\n",
"xc = pairwise(x, corrcoef)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 17.4s\n",
"CPU times: user 2min 59s, sys: 939 ms, total: 3min\n",
"Wall time: 18.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"a65e869c-1180-47af-b8ed-1fef1774ba4b\" data-root-id=\"1002\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"a72b0533-121b-484d-b11e-0eff1772be31\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1013\"}],\"center\":[{\"id\":\"1016\"},{\"id\":\"1020\"},{\"id\":\"1042\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1044\"}},\"left\":[{\"id\":\"1017\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1033\"},{\"id\":\"1048\"}],\"right\":[{\"id\":\"1060\"}],\"title\":{\"id\":\"1003\"},\"toolbar\":{\"id\":\"1025\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1005\"},\"x_scale\":{\"id\":\"1009\"},\"y_range\":{\"id\":\"1007\"},\"y_scale\":{\"id\":\"1011\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1039\"},\"ticker\":{\"id\":\"1018\"}},\"id\":\"1017\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1057\",\"type\":\"Selection\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1046\",\"type\":\"Line\"},{\"attributes\":{\"axis\":{\"id\":\"1017\"},\"dimension\":1,\"ticker\":null},\"id\":\"1020\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1037\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"end\":17.0296717279125},\"id\":\"1005\",\"type\":\"Range1d\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1047\",\"type\":\"Line\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1063\"},\"ticker\":{\"id\":\"1062\"},\"y_range_name\":\"memory\"},\"id\":\"1060\",\"type\":\"LinearAxis\"},{\"attributes\":{\"data_source\":{\"id\":\"1045\"},\"glyph\":{\"id\":\"1046\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1047\"},\"selection_glyph\":null,\"view\":{\"id\":\"1049\"},\"y_range_name\":\"memory\"},\"id\":\"1048\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1048\"}]},\"id\":\"1059\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"BasicTicker\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1021\"},{\"id\":\"1022\"},{\"id\":\"1023\"},{\"id\":\"1024\"}]},\"id\":\"1025\",\"type\":\"Toolbar\"},{\"attributes\":{\"source\":{\"id\":\"1030\"}},\"id\":\"1034\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1037\"},\"ticker\":{\"id\":\"1014\"}},\"id\":\"1013\",\"type\":\"LinearAxis\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1003\",\"type\":\"Title\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1032\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1011\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"LinearScale\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0088732310105115,2.010139918886125,3.010690572904423,4.012023001909256,5.013355346862227,6.014672346878797,7.015997506910935,8.01730971597135,9.018618041882291,10.019920282997191,11.021223830990493,12.022517927922308,13.023813319858164,14.025107317836955,15.026489314856008,16.028091030893847,17.0296717279125],\"y\":[1851.895808,1873.027072,1874.919424,1876.271104,1877.082112,1877.622784,1878.433792,1881.071616,1883.168768,1883.168768,1883.168768,1885.26592,1885.26592,1885.26592,1885.26592,1885.26592,1885.528064,1885.528064]},\"selected\":{\"id\":\"1057\"},\"selection_policy\":{\"id\":\"1058\"}},\"id\":\"1045\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1041\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis\":{\"id\":\"1013\"},\"ticker\":null},\"id\":\"1016\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1040\",\"type\":\"Selection\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1024\",\"type\":\"PanTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1030\"},\"glyph\":{\"id\":\"1031\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1032\"},\"selection_glyph\":null,\"view\":{\"id\":\"1034\"}},\"id\":\"1033\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0088732310105115,2.010139918886125,3.010690572904423,4.012023001909256,5.013355346862227,6.014672346878797,7.015997506910935,8.01730971597135,9.018618041882291,10.019920282997191,11.021223830990493,12.022517927922308,13.023813319858164,14.025107317836955,15.026489314856008,16.028091030893847,17.0296717279125],\"y\":[0.0,1485.8,1592.9,1519.2,1499.0,1499.0,1501.0,1011.7,899.8,901.8,899.8,899.8,902.8,900.8,899.8,812.8,318.5,200.7]},\"selected\":{\"id\":\"1040\"},\"selection_policy\":{\"id\":\"1041\"}},\"id\":\"1030\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"source\":{\"id\":\"1045\"}},\"id\":\"1049\",\"type\":\"CDSView\"},{\"attributes\":{\"end\":1951.895808,\"start\":1851.895808},\"id\":\"1044\",\"type\":\"Range1d\"},{\"attributes\":{\"end\":1592.9},\"id\":\"1007\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1063\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1039\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1062\",\"type\":\"BasicTicker\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1031\",\"type\":\"Line\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1033\"}]},\"id\":\"1043\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1058\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1023\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"items\":[{\"id\":\"1043\"},{\"id\":\"1059\"}]},\"id\":\"1042\",\"type\":\"Legend\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"a72b0533-121b-484d-b11e-0eff1772be31\",\"root_ids\":[\"1002\"],\"roots\":{\"1002\":\"a65e869c-1180-47af-b8ed-1fef1774ba4b\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1002"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1002', <span id=\"1128\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1013', ...)],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1016', ...), Grid(id='1020', ...), Legend(id='1042', ...)],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1044', ...)},</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1017', ...)],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1033', ...), GlyphRenderer(id='1048', ...)],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1060', ...)],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1003', ...),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1025', ...),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1005', ...),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1009', ...),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1007', ...),</div></div><div class=\"1127\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1011', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1128\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1127\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1002', ...)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 3\n",
"\n",
"Use `guvectorize` with vector inputs. This would be the simplest to program new custom functions for."
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 44 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<lambda, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@guvectorize(['void(int8[:], int8[:], float64[:])'], '(n),(n)->()', fastmath=True)\n",
"def corrcoef(x, y, out):\n",
" cov = ((x - x.mean()) * (y - y.mean())).sum() \n",
" out[0] = cov / (x.std() * y.std()) / x.shape[0]\n",
" # out[0] = np.corrcoef(x, y)[0, 1] 2x slower\n",
" \n",
"def pairwise(x, fn):\n",
" return da.blockwise(\n",
" # Note the broadcast here\n",
" lambda x, y: fn(x[:, None, :], y), 'jk', \n",
" x, 'ji',\n",
" x, 'ki', \n",
" dtype='float64',\n",
" concatenate=True\n",
" )\n",
" \n",
"xc = pairwise(x, corrcoef)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 14.2s\n",
"CPU times: user 2min 30s, sys: 1.11 s, total: 2min 31s\n",
"Wall time: 15.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"eb17a28c-ed93-4c8a-a725-333ff3351202\" data-root-id=\"1130\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"9b8fa5d4-a01e-43ee-b8e4-1a3d58899c76\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1141\"}],\"center\":[{\"id\":\"1144\"},{\"id\":\"1148\"},{\"id\":\"1170\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1172\"}},\"left\":[{\"id\":\"1145\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1161\"},{\"id\":\"1176\"}],\"right\":[{\"id\":\"1188\"}],\"title\":{\"id\":\"1131\"},\"toolbar\":{\"id\":\"1153\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1133\"},\"x_scale\":{\"id\":\"1137\"},\"y_range\":{\"id\":\"1135\"},\"y_scale\":{\"id\":\"1139\"}},\"id\":\"1130\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1167\"},\"ticker\":{\"id\":\"1146\"}},\"id\":\"1145\",\"type\":\"LinearAxis\"},{\"attributes\":{\"end\":1565.9},\"id\":\"1135\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1167\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"end\":2020.159744,\"start\":1920.159744},\"id\":\"1172\",\"type\":\"Range1d\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0082230640109628,2.0094767122063786,3.009856587043032,4.0111924591474235,5.012527510058135,6.0138544561341405,7.015170552069321,8.016483541112393,9.017803230090067,10.019116618204862,11.020428589079529,12.021749816136435,13.02306408318691,14.024466552073136],\"y\":[0.0,1464.0,1565.9,1504.4,1497.0,1390.1,902.8,899.8,899.8,902.8,899.8,898.8,903.8,894.8,272.6]},\"selected\":{\"id\":\"1168\"},\"selection_policy\":{\"id\":\"1169\"}},\"id\":\"1158\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1186\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1202\"},\"ticker\":{\"id\":\"1201\"},\"y_range_name\":\"memory\"},\"id\":\"1188\",\"type\":\"LinearAxis\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1176\"}]},\"id\":\"1187\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1150\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1168\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1169\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1165\"},\"ticker\":{\"id\":\"1142\"}},\"id\":\"1141\",\"type\":\"LinearAxis\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1131\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1137\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1146\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1152\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1149\",\"type\":\"SaveTool\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0082230640109628,2.0094767122063786,3.009856587043032,4.0111924591474235,5.012527510058135,6.0138544561341405,7.015170552069321,8.016483541112393,9.017803230090067,10.019116618204862,11.020428589079529,12.021749816136435,13.02306408318691,14.024466552073136],\"y\":[1920.159744,1925.2224,1925.492736,1926.033408,1926.57408,1926.844416,1927.385088,1927.92576,1928.196096,1930.83392,1930.83392,1930.83392,1930.83392,1930.83392,1930.83392]},\"selected\":{\"id\":\"1185\"},\"selection_policy\":{\"id\":\"1186\"}},\"id\":\"1173\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1185\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1173\"},\"glyph\":{\"id\":\"1174\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1175\"},\"selection_glyph\":null,\"view\":{\"id\":\"1177\"},\"y_range_name\":\"memory\"},\"id\":\"1176\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1142\",\"type\":\"BasicTicker\"},{\"attributes\":{\"source\":{\"id\":\"1158\"}},\"id\":\"1162\",\"type\":\"CDSView\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1174\",\"type\":\"Line\"},{\"attributes\":{\"items\":[{\"id\":\"1171\"},{\"id\":\"1187\"}]},\"id\":\"1170\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"1139\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1202\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"end\":14.024466552073136},\"id\":\"1133\",\"type\":\"Range1d\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1160\",\"type\":\"Line\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1161\"}]},\"id\":\"1171\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1165\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1151\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1159\",\"type\":\"Line\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1149\"},{\"id\":\"1150\"},{\"id\":\"1151\"},{\"id\":\"1152\"}]},\"id\":\"1153\",\"type\":\"Toolbar\"},{\"attributes\":{\"source\":{\"id\":\"1173\"}},\"id\":\"1177\",\"type\":\"CDSView\"},{\"attributes\":{\"axis\":{\"id\":\"1145\"},\"dimension\":1,\"ticker\":null},\"id\":\"1148\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1201\",\"type\":\"BasicTicker\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1175\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"1158\"},\"glyph\":{\"id\":\"1159\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1160\"},\"selection_glyph\":null,\"view\":{\"id\":\"1162\"}},\"id\":\"1161\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis\":{\"id\":\"1141\"},\"ticker\":null},\"id\":\"1144\",\"type\":\"Grid\"}],\"root_ids\":[\"1130\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"9b8fa5d4-a01e-43ee-b8e4-1a3d58899c76\",\"root_ids\":[\"1130\"],\"roots\":{\"1130\":\"eb17a28c-ed93-4c8a-a725-333ff3351202\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1130"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1130', <span id=\"1267\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1141', ...)],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1144', ...), Grid(id='1148', ...), Legend(id='1170', ...)],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1172', ...)},</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1145', ...)],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1161', ...), GlyphRenderer(id='1176', ...)],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1188', ...)],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1131', ...),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1153', ...),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1133', ...),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1137', ...),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1135', ...),</div></div><div class=\"1266\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1139', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1267\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1266\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1130', ...)"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Materialize array broadcast\n",
"\n",
"As a control for memory usage, compare the above to a similar implementation that forces the broadcasted chunks to materialize."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 44 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<lambda, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@guvectorize(['void(int8[:], int8[:], float64[:])'], '(n),(n)->()', fastmath=True)\n",
"def corrcoef(x, y, out):\n",
" cov = ((x - x.mean()) * (y - y.mean())).sum() \n",
" out[0] = cov / (x.std() * y.std()) / x.shape[0]\n",
"\n",
" \n",
"def pairwise(x, fn):\n",
" return da.blockwise(\n",
" # np.array necessary to create actual array from views\n",
" lambda x, y: fn(*[np.array(a) for a in np.broadcast_arrays(x[:, None, :], y)]), 'jk', \n",
" x, 'ji',\n",
" x, 'ki', \n",
" dtype='float64',\n",
" concatenate=True\n",
" )\n",
" \n",
"xc = pairwise(x, corrcoef)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 41.9s\n",
"CPU times: user 6min 52s, sys: 14.7 s, total: 7min 6s\n",
"Wall time: 42.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Memory usage here is WAY higher -- 35G vs 1.4G:"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"e4fb1899-fe85-4413-979f-039b74e18052\" data-root-id=\"1269\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"0c34d0fe-c536-4482-96ee-64527b6dd527\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1280\"}],\"center\":[{\"id\":\"1283\"},{\"id\":\"1287\"},{\"id\":\"1309\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1311\"}},\"left\":[{\"id\":\"1284\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1300\"},{\"id\":\"1315\"}],\"right\":[{\"id\":\"1327\"}],\"title\":{\"id\":\"1270\"},\"toolbar\":{\"id\":\"1292\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1272\"},\"x_scale\":{\"id\":\"1276\"},\"y_range\":{\"id\":\"1274\"},\"y_scale\":{\"id\":\"1278\"}},\"id\":\"1269\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0139304189942777,2.015550035983324,3.017108154948801,4.01862131501548,5.019536499166861,6.021037944126874,7.022537128068507,8.02351320697926,9.02497257408686,10.026440653018653,11.02752647199668,12.029020497109741,13.03053791099228,14.031533506000414,15.033530754037201,16.03499642503448,17.036435229005292,18.0378488691058,19.039237768156454,20.040670993970707,21.042116277152672,22.043553662020713,23.044987933011726,24.046422012150288,25.04752314114012,26.048956179060042,27.050363142043352,28.051769317127764,29.053199412068352,30.054613156011328,31.056027990067378,32.057414966169745,33.0588159491308,34.059513251064345,35.061004378134385,36.06264564907178,37.06427900097333,38.06591470609419,39.06756498198956,40.06917700706981,41.0707836849615],\"y\":[0.0,1467.6,1589.4,1594.6,1592.6,1598.5,1594.6,1520.7,1505.6,1498.8,1497.8,1505.4,1498.7,1498.7,1413.9,1365.0,1299.1,1223.3,899.7,899.8,903.7,899.7,903.7,899.7,899.7,905.0,899.7,899.7,903.7,899.7,900.7,902.7,899.8,899.7,835.4,512.2,200.7,202.7,200.7,202.7,201.7,201.7]},\"selected\":{\"id\":\"1307\"},\"selection_policy\":{\"id\":\"1308\"}},\"id\":\"1297\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1314\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1278\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1304\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1308\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1306\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1276\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1288\",\"type\":\"SaveTool\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1299\",\"type\":\"Line\"},{\"attributes\":{\"source\":{\"id\":\"1297\"}},\"id\":\"1301\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1352\"},\"ticker\":{\"id\":\"1351\"},\"y_range_name\":\"memory\"},\"id\":\"1327\",\"type\":\"LinearAxis\"},{\"attributes\":{\"axis\":{\"id\":\"1284\"},\"dimension\":1,\"ticker\":null},\"id\":\"1287\",\"type\":\"Grid\"},{\"attributes\":{\"items\":[{\"id\":\"1310\"},{\"id\":\"1326\"}]},\"id\":\"1309\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"1351\",\"type\":\"BasicTicker\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1315\"}]},\"id\":\"1326\",\"type\":\"LegendItem\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1304\"},\"ticker\":{\"id\":\"1281\"}},\"id\":\"1280\",\"type\":\"LinearAxis\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1313\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1352\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1285\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1306\"},\"ticker\":{\"id\":\"1285\"}},\"id\":\"1284\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1281\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1307\",\"type\":\"Selection\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0139304189942777,2.015550035983324,3.017108154948801,4.01862131501548,5.019536499166861,6.021037944126874,7.022537128068507,8.02351320697926,9.02497257408686,10.026440653018653,11.02752647199668,12.029020497109741,13.03053791099228,14.031533506000414,15.033530754037201,16.03499642503448,17.036435229005292,18.0378488691058,19.039237768156454,20.040670993970707,21.042116277152672,22.043553662020713,23.044987933011726,24.046422012150288,25.04752314114012,26.048956179060042,27.050363142043352,28.051769317127764,29.053199412068352,30.054613156011328,31.056027990067378,32.057414966169745,33.0588159491308,34.059513251064345,35.061004378134385,36.06264564907178,37.06427900097333,38.06591470609419,39.06756498198956,40.06917700706981,41.0707836849615],\"y\":[1941.884928,20141.428736,37885.41952,37949.198336,37949.415424,37949.68576,37951.782912,37591.77728,37591.77728,37591.77728,37591.965696,37591.965696,37592.219648,37592.219648,36512.489472,35432.48896,35432.910848,31113.396224,31115.763712,31116.034048,31116.304384,31116.57472,31116.845056,31118.942208,31118.942208,31118.942208,31118.942208,31118.942208,31119.204352,31119.474688,31119.745024,31120.125952,31120.396288,31120.666624,24640.933888,8441.147392,8441.147392,8441.147392,8441.147392,8441.147392,8441.147392,8441.147392]},\"selected\":{\"id\":\"1324\"},\"selection_policy\":{\"id\":\"1325\"}},\"id\":\"1312\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1289\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1324\",\"type\":\"Selection\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1291\",\"type\":\"PanTool\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1300\"}]},\"id\":\"1310\",\"type\":\"LegendItem\"},{\"attributes\":{\"axis\":{\"id\":\"1280\"},\"ticker\":null},\"id\":\"1283\",\"type\":\"Grid\"},{\"attributes\":{\"end\":41.0707836849615},\"id\":\"1272\",\"type\":\"Range1d\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1298\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1325\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"data_source\":{\"id\":\"1312\"},\"glyph\":{\"id\":\"1313\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1314\"},\"selection_glyph\":null,\"view\":{\"id\":\"1316\"},\"y_range_name\":\"memory\"},\"id\":\"1315\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1288\"},{\"id\":\"1289\"},{\"id\":\"1290\"},{\"id\":\"1291\"}]},\"id\":\"1292\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1297\"},\"glyph\":{\"id\":\"1298\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1299\"},\"selection_glyph\":null,\"view\":{\"id\":\"1301\"}},\"id\":\"1300\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"end\":37951.782912,\"start\":1941.884928},\"id\":\"1311\",\"type\":\"Range1d\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1270\",\"type\":\"Title\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1290\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"source\":{\"id\":\"1312\"}},\"id\":\"1316\",\"type\":\"CDSView\"},{\"attributes\":{\"end\":1598.5},\"id\":\"1274\",\"type\":\"Range1d\"}],\"root_ids\":[\"1269\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"0c34d0fe-c536-4482-96ee-64527b6dd527\",\"root_ids\":[\"1269\"],\"roots\":{\"1269\":\"e4fb1899-fe85-4413-979f-039b74e18052\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1269"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1269', <span id=\"1417\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1280', ...)],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1283', ...), Grid(id='1287', ...), Legend(id='1309', ...)],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1311', ...)},</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1284', ...)],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1300', ...), GlyphRenderer(id='1315', ...)],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1327', ...)],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1270', ...),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1292', ...),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1272', ...),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1276', ...),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1274', ...),</div></div><div class=\"1416\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1278', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1417\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1416\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1269', ...)"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 4\n",
"\n",
"Use `njit` and `cuda.jit` to program separate functions. This would be the hardest to maintain."
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 72 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<corrcoef, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@njit(fastmath=True, nogil=True) # nogil crucial here\n",
"def corrcoef(x, y):\n",
" n, m = x.shape[0], y.shape[1]\n",
" out = np.zeros((n, m), dtype=np.float64)\n",
" \n",
" def compute(xv, yv):\n",
" xvc = (xv - xv.mean()) / np.std(xv)\n",
" yvc = (yv - yv.mean()) / np.std(yv)\n",
" return (xvc * yvc).mean()\n",
" \n",
" for i in range(n):\n",
" for j in range(m):\n",
" out[i, j] = compute(x[i], y[:, j])\n",
" # out[i, j] = np.corrcoef(x[i], y[:, j])[0, 1]\n",
" return out\n",
"\n",
"def pairwise(x, fn):\n",
" return da.blockwise(\n",
" fn, 'jk', \n",
" x, 'ji',\n",
" x.T, 'ik', \n",
" dtype='float64',\n",
" concatenate=True\n",
" )\n",
"\n",
"xc = pairwise(x, corrcoef)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 14.5s\n",
"CPU times: user 2min 30s, sys: 1.05 s, total: 2min 31s\n",
"Wall time: 15.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"6ebb8fd8-d587-437d-bd65-609f90653632\" data-root-id=\"1419\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"d2c6604b-3947-4a82-bccc-0a9ba54c63c2\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1430\"}],\"center\":[{\"id\":\"1433\"},{\"id\":\"1437\"},{\"id\":\"1459\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1461\"}},\"left\":[{\"id\":\"1434\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1450\"},{\"id\":\"1465\"}],\"right\":[{\"id\":\"1477\"}],\"title\":{\"id\":\"1420\"},\"toolbar\":{\"id\":\"1442\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1422\"},\"x_scale\":{\"id\":\"1426\"},\"y_range\":{\"id\":\"1424\"},\"y_scale\":{\"id\":\"1428\"}},\"id\":\"1419\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1454\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1428\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1426\",\"type\":\"LinearScale\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1438\"},{\"id\":\"1439\"},{\"id\":\"1440\"},{\"id\":\"1441\"}]},\"id\":\"1442\",\"type\":\"Toolbar\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1464\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1475\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0083657980430871,2.0096396699082106,3.0104152609128505,4.011453077895567,5.01246567000635,6.013802808942273,7.015127751976252,8.016333847073838,9.017650168854743,10.01896010292694,11.02028727484867,12.021600121865049,13.022905617952347,14.024327045073733],\"y\":[0.0,1474.8,1571.9,1499.8,1499.4,1438.5,967.7,899.8,902.9,899.8,899.8,903.8,898.8,774.0,210.7]},\"selected\":{\"id\":\"1457\"},\"selection_policy\":{\"id\":\"1458\"}},\"id\":\"1447\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"data_source\":{\"id\":\"1447\"},\"glyph\":{\"id\":\"1448\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1449\"},\"selection_glyph\":null,\"view\":{\"id\":\"1451\"}},\"id\":\"1450\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1438\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1431\",\"type\":\"BasicTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"1462\"},\"glyph\":{\"id\":\"1463\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1464\"},\"selection_glyph\":null,\"view\":{\"id\":\"1466\"},\"y_range_name\":\"memory\"},\"id\":\"1465\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1463\",\"type\":\"Line\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1449\",\"type\":\"Line\"},{\"attributes\":{\"axis\":{\"id\":\"1434\"},\"dimension\":1,\"ticker\":null},\"id\":\"1437\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1462\"}},\"id\":\"1466\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1456\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1513\"},\"ticker\":{\"id\":\"1512\"},\"y_range_name\":\"memory\"},\"id\":\"1477\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1513\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0083657980430871,2.0096396699082106,3.0104152609128505,4.011453077895567,5.01246567000635,6.013802808942273,7.015127751976252,8.016333847073838,9.017650168854743,10.01896010292694,11.02028727484867,12.021600121865049,13.022905617952347,14.024327045073733],\"y\":[1980.104704,1984.155648,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776,1984.75776]},\"selected\":{\"id\":\"1474\"},\"selection_policy\":{\"id\":\"1475\"}},\"id\":\"1462\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1456\"},\"ticker\":{\"id\":\"1435\"}},\"id\":\"1434\",\"type\":\"LinearAxis\"},{\"attributes\":{\"axis\":{\"id\":\"1430\"},\"ticker\":null},\"id\":\"1433\",\"type\":\"Grid\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1450\"}]},\"id\":\"1460\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1512\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1435\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1454\"},\"ticker\":{\"id\":\"1431\"}},\"id\":\"1430\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1439\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1457\",\"type\":\"Selection\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1465\"}]},\"id\":\"1476\",\"type\":\"LegendItem\"},{\"attributes\":{\"end\":1571.9},\"id\":\"1424\",\"type\":\"Range1d\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1440\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1474\",\"type\":\"Selection\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1420\",\"type\":\"Title\"},{\"attributes\":{\"items\":[{\"id\":\"1460\"},{\"id\":\"1476\"}]},\"id\":\"1459\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"1458\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"source\":{\"id\":\"1447\"}},\"id\":\"1451\",\"type\":\"CDSView\"},{\"attributes\":{\"end\":2080.1047040000003,\"start\":1980.104704},\"id\":\"1461\",\"type\":\"Range1d\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1441\",\"type\":\"PanTool\"},{\"attributes\":{\"end\":14.024327045073733},\"id\":\"1422\",\"type\":\"Range1d\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1448\",\"type\":\"Line\"}],\"root_ids\":[\"1419\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"d2c6604b-3947-4a82-bccc-0a9ba54c63c2\",\"root_ids\":[\"1419\"],\"roots\":{\"1419\":\"6ebb8fd8-d587-437d-bd65-609f90653632\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1419"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1419', <span id=\"1578\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1430', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1433', ...), Grid(id='1437', ...), Legend(id='1459', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1461', ...)},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1434', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1450', ...), GlyphRenderer(id='1465', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1477', ...)],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1420', ...),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1442', ...),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1422', ...),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1426', ...),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1424', ...),</div></div><div class=\"1577\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1428', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1578\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1577\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1419', ...)"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 5\n",
"\n",
"Fully block-scalable pairwise implementation from https://github.com/pystatgen/sgkit/issues/241#issuecomment-698878378.\n",
" \n",
"This is quite slow by comparison, and may possibly be materializing broadcasted blocks."
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 822 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<transpose, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@guvectorize(['void(int8[:], int8[:], float64[:], float64[:])'], '(n),(n),(p)->(p)')\n",
"def corr_map(x, y, _, out):\n",
" out[:] = np.array([\n",
" np.sum(x),\n",
" np.sum(y),\n",
" np.sum(x * x),\n",
" np.sum(y * y),\n",
" np.sum(x * y),\n",
" len(x)\n",
" ])\n",
"\n",
"\n",
"@guvectorize(['void(float64[:,:], float64[:])'], '(p,m)->()')\n",
"def corr_reduce(v, out):\n",
" v = v.sum(axis=-1)\n",
" n = v[5]\n",
" num = n * v[4] - v[0] * v[1]\n",
" denom1 = np.sqrt(n * v[2] - v[0]**2)\n",
" denom2 = np.sqrt(n * v[3] - v[1]**2)\n",
" out[0] = num / (denom1 * denom2)\n",
"\n",
"\n",
"def pairwise(x, map_fn, reduce_fn, n_map_param):\n",
" x_map = da.concatenate([\n",
" da.concatenate([\n",
" da.stack([\n",
" map_fn(\n",
" x.blocks[i1, j][:, None, :],\n",
" x.blocks[i2, j],\n",
" np.empty(n_map_param)\n",
" )\n",
" for j in range(x.numblocks[1])\n",
" ], axis=-1)\n",
" for i2 in range(x.numblocks[0])\n",
" ], axis=1)\n",
" for i1 in range(x.numblocks[0])\n",
" ], axis=0)\n",
" assert x_map.shape == (len(x), len(x), n_map_param, x.numblocks[1])\n",
"\n",
" x_reduce = reduce_fn(x_map.rechunk((None, None, -1, -1)))\n",
" return x_reduce\n",
"\n",
"\n",
"# There are 6 intermediate values in this case\n",
"xc = pairwise(x, corr_map, corr_reduce, 6)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 39.7s\n",
"CPU times: user 10min 12s, sys: 2.39 s, total: 10min 14s\n",
"Wall time: 40.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"52ed8cc5-985e-4df7-8046-5d2cfdde6681\" data-root-id=\"1580\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"3ed9088a-9951-4c62-88dc-519b7e641c0d\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1591\"}],\"center\":[{\"id\":\"1594\"},{\"id\":\"1598\"},{\"id\":\"1620\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1622\"}},\"left\":[{\"id\":\"1595\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1611\"},{\"id\":\"1626\"}],\"right\":[{\"id\":\"1638\"}],\"title\":{\"id\":\"1581\"},\"toolbar\":{\"id\":\"1603\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1583\"},\"x_scale\":{\"id\":\"1587\"},\"y_range\":{\"id\":\"1585\"},\"y_scale\":{\"id\":\"1589\"}},\"id\":\"1580\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"end\":4610.584576,\"start\":1994.592256},\"id\":\"1622\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1599\",\"type\":\"SaveTool\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1626\"}]},\"id\":\"1637\",\"type\":\"LegendItem\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1610\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"1623\"},\"glyph\":{\"id\":\"1624\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1625\"},\"selection_glyph\":null,\"view\":{\"id\":\"1627\"},\"y_range_name\":\"memory\"},\"id\":\"1626\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1611\"}]},\"id\":\"1621\",\"type\":\"LegendItem\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1599\"},{\"id\":\"1600\"},{\"id\":\"1601\"},{\"id\":\"1602\"}]},\"id\":\"1603\",\"type\":\"Toolbar\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1624\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1684\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1591\"},\"ticker\":null},\"id\":\"1594\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1635\",\"type\":\"Selection\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.009261725936085,2.0105035109445453,3.010815816000104,4.012122112093493,5.013423181138933,6.014708354137838,7.016004756093025,8.017316682031378,9.018644794123247,10.019952466944233,11.021274527069181,12.022578089032322,13.023899069987237,14.025209835032001,15.026513054035604,16.027843018062413,17.0291481311433,18.030457724118605,19.030814989935607,20.032107705017552,21.03340388997458,22.034703942015767,23.036024595145136,24.037339431932196,25.0386527441442,26.03997110412456,27.041275799972937,28.04256593505852,29.04385856213048,30.045153886079788,31.046471195062622,32.0468134470284,33.048120273975655,34.04941941215657,35.050741811981425,36.05207387008704,37.053404178936034,38.05473667010665,39.056072316132486],\"y\":[0.0,1328.9,1592.9,1603.5,1593.9,1593.9,1601.9,1593.0,1589.9,1601.9,1595.9,1592.9,1595.9,1598.9,1594.9,1593.9,1598.9,1595.9,1593.9,1601.4,1595.0,1591.9,1597.9,1593.9,1593.9,1591.9,1594.9,1592.9,1593.0,1601.9,1593.9,1593.9,1602.4,1592.9,1594.9,1590.9,1591.9,1575.9,1144.5,941.7]},\"selected\":{\"id\":\"1618\"},\"selection_policy\":{\"id\":\"1619\"}},\"id\":\"1608\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1618\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1587\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1615\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1581\",\"type\":\"Title\"},{\"attributes\":{\"source\":{\"id\":\"1608\"}},\"id\":\"1612\",\"type\":\"CDSView\"},{\"attributes\":{\"source\":{\"id\":\"1623\"}},\"id\":\"1627\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1617\"},\"ticker\":{\"id\":\"1596\"}},\"id\":\"1595\",\"type\":\"LinearAxis\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1602\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1619\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"end\":1603.5},\"id\":\"1585\",\"type\":\"Range1d\"},{\"attributes\":{\"axis\":{\"id\":\"1595\"},\"dimension\":1,\"ticker\":null},\"id\":\"1598\",\"type\":\"Grid\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1625\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1636\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1592\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1615\"},\"ticker\":{\"id\":\"1592\"}},\"id\":\"1591\",\"type\":\"LinearAxis\"},{\"attributes\":{\"items\":[{\"id\":\"1621\"},{\"id\":\"1637\"}]},\"id\":\"1620\",\"type\":\"Legend\"},{\"attributes\":{\"data_source\":{\"id\":\"1608\"},\"glyph\":{\"id\":\"1609\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1610\"},\"selection_glyph\":null,\"view\":{\"id\":\"1612\"}},\"id\":\"1611\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1685\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1600\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1589\",\"type\":\"LinearScale\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1609\",\"type\":\"Line\"},{\"attributes\":{\"end\":39.056072316132486},\"id\":\"1583\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1596\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1601\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1685\"},\"ticker\":{\"id\":\"1684\"},\"y_range_name\":\"memory\"},\"id\":\"1638\",\"type\":\"LinearAxis\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.009261725936085,2.0105035109445453,3.010815816000104,4.012122112093493,5.013423181138933,6.014708354137838,7.016004756093025,8.017316682031378,9.018644794123247,10.019952466944233,11.021274527069181,12.022578089032322,13.023899069987237,14.025209835032001,15.026513054035604,16.027843018062413,17.0291481311433,18.030457724118605,19.030814989935607,20.032107705017552,21.03340388997458,22.034703942015767,23.036024595145136,24.037339431932196,25.0386527441442,26.03997110412456,27.041275799972937,28.04256593505852,29.04385856213048,30.045153886079788,31.046471195062622,32.0468134470284,33.048120273975655,34.04941941215657,35.050741811981425,36.05207387008704,37.053404178936034,38.05473667010665,39.056072316132486],\"y\":[1994.592256,2030.338048,2070.437888,2106.089472,2146.172928,2179.72736,2221.6704,2253.594624,3113.058304,2950.496256,3024.0768,3097.751552,3177.6768,2433.548288,2517.639168,2583.842816,2643.423232,2691.657728,2750.619648,2803.048448,2858.594304,2918.629376,3523.026944,3596.120064,3661.131776,3739.295744,3806.37184,2836.627456,2912.985088,2984.697856,3047.58784,3368.2432,3412.74624,3464.94976,3938.586624,4121.288704,4253.290496,4332.658688,4610.584576,3925.327872]},\"selected\":{\"id\":\"1635\"},\"selection_policy\":{\"id\":\"1636\"}},\"id\":\"1623\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1617\",\"type\":\"BasicTickFormatter\"}],\"root_ids\":[\"1580\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"3ed9088a-9951-4c62-88dc-519b7e641c0d\",\"root_ids\":[\"1580\"],\"roots\":{\"1580\":\"52ed8cc5-985e-4df7-8046-5d2cfdde6681\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1580"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1580', <span id=\"1750\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1591', ...)],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1594', ...), Grid(id='1598', ...), Legend(id='1620', ...)],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1622', ...)},</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1595', ...)],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1611', ...), GlyphRenderer(id='1626', ...)],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1638', ...)],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1581', ...),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1603', ...),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1583', ...),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1587', ...),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1585', ...),</div></div><div class=\"1749\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1589', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1750\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1749\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1580', ...)"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Option 6\n",
"\n",
"Same as option 5 but with column chunking removed as `da.blockwise` does implicitly."
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
" <thead>\n",
" <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr><th> Bytes </th><td> 72.00 MB </td> <td> 6.48 MB </td></tr>\n",
" <tr><th> Shape </th><td> (3000, 3000) </td> <td> (900, 900) </td></tr>\n",
" <tr><th> Count </th><td> 185 Tasks </td><td> 16 Chunks </td></tr>\n",
" <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
" </tbody>\n",
"</table>\n",
"</td>\n",
"<td>\n",
"<svg width=\"170\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
"\n",
" <!-- Horizontal lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"120\" y2=\"0\" style=\"stroke-width:2\" />\n",
" <line x1=\"0\" y1=\"36\" x2=\"120\" y2=\"36\" />\n",
" <line x1=\"0\" y1=\"72\" x2=\"120\" y2=\"72\" />\n",
" <line x1=\"0\" y1=\"108\" x2=\"120\" y2=\"108\" />\n",
" <line x1=\"0\" y1=\"120\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Vertical lines -->\n",
" <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
" <line x1=\"36\" y1=\"0\" x2=\"36\" y2=\"120\" />\n",
" <line x1=\"72\" y1=\"0\" x2=\"72\" y2=\"120\" />\n",
" <line x1=\"108\" y1=\"0\" x2=\"108\" y2=\"120\" />\n",
" <line x1=\"120\" y1=\"0\" x2=\"120\" y2=\"120\" style=\"stroke-width:2\" />\n",
"\n",
" <!-- Colored Rectangle -->\n",
" <polygon points=\"0.000000,0.000000 120.000000,0.000000 120.000000,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
"\n",
" <!-- Text -->\n",
" <text x=\"60.000000\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3000</text>\n",
" <text x=\"140.000000\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,140.000000,60.000000)\">3000</text>\n",
"</svg>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"dask.array<transpose, shape=(3000, 3000), dtype=float64, chunksize=(900, 900), chunktype=numpy.ndarray>"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# There are 6 intermediate values in this case\n",
"xc = pairwise(x.rechunk((None, -1)), corr_map, corr_reduce, 6)\n",
"xc"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[########################################] | 100% Completed | 12.7s\n",
"CPU times: user 2min 16s, sys: 3.18 s, total: 2min 19s\n",
"Wall time: 13.5 s\n"
]
}
],
"source": [
"%%time\n",
"with ProgressBar(), ResourceProfiler() as prof:\n",
" c1 = xc.compute()\n",
"c2 = np.corrcoef(x.compute())\n",
"np.testing.assert_allclose(c1, c2, atol=1e-8)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" <div class=\"bk-root\" id=\"4a5f17fc-b23a-40a2-ad3d-88c4291e230c\" data-root-id=\"1752\"></div>\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/javascript": [
"(function(root) {\n",
" function embed_document(root) {\n",
" \n",
" var docs_json = {\"962d97dc-fc7a-4b1b-ba7c-563e2a52a9b1\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1763\"}],\"center\":[{\"id\":\"1766\"},{\"id\":\"1770\"},{\"id\":\"1792\"}],\"extra_y_ranges\":{\"memory\":{\"id\":\"1794\"}},\"left\":[{\"id\":\"1767\"}],\"plot_height\":300,\"plot_width\":800,\"renderers\":[{\"id\":\"1783\"},{\"id\":\"1798\"}],\"right\":[{\"id\":\"1810\"}],\"title\":{\"id\":\"1753\"},\"toolbar\":{\"id\":\"1775\"},\"toolbar_location\":\"above\",\"x_range\":{\"id\":\"1755\"},\"x_scale\":{\"id\":\"1759\"},\"y_range\":{\"id\":\"1757\"},\"y_scale\":{\"id\":\"1761\"}},\"id\":\"1752\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1796\",\"type\":\"Line\"},{\"attributes\":{\"axis_label\":\"% CPU\",\"formatter\":{\"id\":\"1789\"},\"ticker\":{\"id\":\"1768\"}},\"id\":\"1767\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1789\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1772\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1787\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1774\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1868\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"text\":\"Profile Results\"},\"id\":\"1753\",\"type\":\"Title\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1782\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1791\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"end\":1548.9},\"id\":\"1757\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1771\",\"type\":\"SaveTool\"},{\"attributes\":{\"axis\":{\"id\":\"1767\"},\"dimension\":1,\"ticker\":null},\"id\":\"1770\",\"type\":\"Grid\"},{\"attributes\":{\"line_color\":\"#440154\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1781\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1761\",\"type\":\"LinearScale\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1773\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#29788E\",\"line_width\":4,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1797\",\"type\":\"Line\"},{\"attributes\":{\"axis_label\":\"Time (s)\",\"formatter\":{\"id\":\"1787\"},\"ticker\":{\"id\":\"1764\"}},\"id\":\"1763\",\"type\":\"LinearAxis\"},{\"attributes\":{\"label\":{\"value\":\"Memory\"},\"renderers\":[{\"id\":\"1798\"}]},\"id\":\"1809\",\"type\":\"LegendItem\"},{\"attributes\":{\"label\":{\"value\":\"% CPU\"},\"renderers\":[{\"id\":\"1783\"}]},\"id\":\"1793\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1807\",\"type\":\"Selection\"},{\"attributes\":{\"end\":12.022843171143904},\"id\":\"1755\",\"type\":\"Range1d\"},{\"attributes\":{\"data_source\":{\"id\":\"1780\"},\"glyph\":{\"id\":\"1781\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1782\"},\"selection_glyph\":null,\"view\":{\"id\":\"1784\"}},\"id\":\"1783\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1808\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"items\":[{\"id\":\"1793\"},{\"id\":\"1809\"}]},\"id\":\"1792\",\"type\":\"Legend\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1771\"},{\"id\":\"1772\"},{\"id\":\"1773\"},{\"id\":\"1774\"}]},\"id\":\"1775\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1768\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1759\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1867\",\"type\":\"BasicTicker\"},{\"attributes\":{\"source\":{\"id\":\"1780\"}},\"id\":\"1784\",\"type\":\"CDSView\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0083158330526203,2.009578129975125,3.010916484054178,4.012259372044355,5.013600699137896,6.014923662180081,7.016248814063147,8.017572743119672,9.018883702112362,10.020201727980748,11.021523302188143,12.022843171143904],\"y\":[2518.79424,2535.030784,2549.71904,2562.301952,2574.884864,2585.370624,2604.244992,2623.11936,2638.000128,2798.288896,2837.078016,2837.078016,2720.497664]},\"selected\":{\"id\":\"1807\"},\"selection_policy\":{\"id\":\"1808\"}},\"id\":\"1795\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"end\":2837.078016,\"start\":2518.79424},\"id\":\"1794\",\"type\":\"Range1d\"},{\"attributes\":{\"data\":{\"x\":[0.0,1.0083158330526203,2.009578129975125,3.010916484054178,4.012259372044355,5.013600699137896,6.014923662180081,7.016248814063147,8.017572743119672,9.018883702112362,10.020201727980748,11.021523302188143,12.022843171143904],\"y\":[0.0,1481.0,1548.9,1504.0,1490.0,1320.2,903.8,899.8,899.8,896.8,887.8,890.8,745.0]},\"selected\":{\"id\":\"1790\"},\"selection_policy\":{\"id\":\"1791\"}},\"id\":\"1780\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1790\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1764\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis_label\":\"Memory (MB)\",\"formatter\":{\"id\":\"1868\"},\"ticker\":{\"id\":\"1867\"},\"y_range_name\":\"memory\"},\"id\":\"1810\",\"type\":\"LinearAxis\"},{\"attributes\":{\"data_source\":{\"id\":\"1795\"},\"glyph\":{\"id\":\"1796\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1797\"},\"selection_glyph\":null,\"view\":{\"id\":\"1799\"},\"y_range_name\":\"memory\"},\"id\":\"1798\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis\":{\"id\":\"1763\"},\"ticker\":null},\"id\":\"1766\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1795\"}},\"id\":\"1799\",\"type\":\"CDSView\"}],\"root_ids\":[\"1752\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.1\"}};\n",
" var render_items = [{\"docid\":\"962d97dc-fc7a-4b1b-ba7c-563e2a52a9b1\",\"root_ids\":[\"1752\"],\"roots\":{\"1752\":\"4a5f17fc-b23a-40a2-ad3d-88c4291e230c\"}}];\n",
" root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
"\n",
" }\n",
" if (root.Bokeh !== undefined) {\n",
" embed_document(root);\n",
" } else {\n",
" var attempts = 0;\n",
" var timer = setInterval(function(root) {\n",
" if (root.Bokeh !== undefined) {\n",
" clearInterval(timer);\n",
" embed_document(root);\n",
" } else {\n",
" attempts++;\n",
" if (attempts > 100) {\n",
" clearInterval(timer);\n",
" console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
" }\n",
" }\n",
" }, 10, root)\n",
" }\n",
"})(window);"
],
"application/vnd.bokehjs_exec.v0+json": ""
},
"metadata": {
"application/vnd.bokehjs_exec.v0+json": {
"id": "1752"
}
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div style=\"display: table;\"><div style=\"display: table-row;\"><div style=\"display: table-cell;\"><b title=\"bokeh.plotting.figure.Figure\">Figure</b>(</div><div style=\"display: table-cell;\">id&nbsp;=&nbsp;'1752', <span id=\"1933\" style=\"cursor: pointer;\">&hellip;)</span></div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">above&nbsp;=&nbsp;[],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">align&nbsp;=&nbsp;'start',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_ratio&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">aspect_scale&nbsp;=&nbsp;1,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">background_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">below&nbsp;=&nbsp;[LinearAxis(id='1763', ...)],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">border_fill_color&nbsp;=&nbsp;'#ffffff',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">center&nbsp;=&nbsp;[Grid(id='1766', ...), Grid(id='1770', ...), Legend(id='1792', ...)],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">css_classes&nbsp;=&nbsp;[],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">disabled&nbsp;=&nbsp;False,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_x_ranges&nbsp;=&nbsp;{},</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">extra_y_ranges&nbsp;=&nbsp;{'memory': Range1d(id='1794', ...)},</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_height&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">frame_width&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">height_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">hidpi&nbsp;=&nbsp;True,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_event_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">js_property_callbacks&nbsp;=&nbsp;{},</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">left&nbsp;=&nbsp;[LinearAxis(id='1767', ...)],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_factor&nbsp;=&nbsp;10,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_interval&nbsp;=&nbsp;300,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_threshold&nbsp;=&nbsp;2000,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">lod_timeout&nbsp;=&nbsp;500,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">margin&nbsp;=&nbsp;(0, 0, 0, 0),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">match_aspect&nbsp;=&nbsp;False,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_height&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">max_width&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border&nbsp;=&nbsp;5,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_bottom&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_left&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_right&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_border_top&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_height&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">min_width&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">name&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_alpha&nbsp;=&nbsp;1.0,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_cap&nbsp;=&nbsp;'butt',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_color&nbsp;=&nbsp;'#e5e5e5',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash&nbsp;=&nbsp;[],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_dash_offset&nbsp;=&nbsp;0,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_join&nbsp;=&nbsp;'bevel',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">outline_line_width&nbsp;=&nbsp;1,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">output_backend&nbsp;=&nbsp;'canvas',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_height&nbsp;=&nbsp;300,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">plot_width&nbsp;=&nbsp;800,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">renderers&nbsp;=&nbsp;[GlyphRenderer(id='1783', ...), GlyphRenderer(id='1798', ...)],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">reset_policy&nbsp;=&nbsp;'standard',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">right&nbsp;=&nbsp;[LinearAxis(id='1810', ...)],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">sizing_mode&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">subscribed_events&nbsp;=&nbsp;[],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">tags&nbsp;=&nbsp;[],</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title&nbsp;=&nbsp;Title(id='1753', ...),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">title_location&nbsp;=&nbsp;'above',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar&nbsp;=&nbsp;Toolbar(id='1775', ...),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_location&nbsp;=&nbsp;'above',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">toolbar_sticky&nbsp;=&nbsp;True,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">visible&nbsp;=&nbsp;True,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width&nbsp;=&nbsp;None,</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">width_policy&nbsp;=&nbsp;'auto',</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_range&nbsp;=&nbsp;Range1d(id='1755', ...),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">x_scale&nbsp;=&nbsp;LinearScale(id='1759', ...),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_range&nbsp;=&nbsp;Range1d(id='1757', ...),</div></div><div class=\"1932\" style=\"display: none;\"><div style=\"display: table-cell;\"></div><div style=\"display: table-cell;\">y_scale&nbsp;=&nbsp;LinearScale(id='1761', ...))</div></div></div>\n",
"<script>\n",
"(function() {\n",
" var expanded = false;\n",
" var ellipsis = document.getElementById(\"1933\");\n",
" ellipsis.addEventListener(\"click\", function() {\n",
" var rows = document.getElementsByClassName(\"1932\");\n",
" for (var i = 0; i < rows.length; i++) {\n",
" var el = rows[i];\n",
" el.style.display = expanded ? \"none\" : \"table-row\";\n",
" }\n",
" ellipsis.innerHTML = expanded ? \"&hellip;)\" : \"&lsaquo;&lsaquo;&lsaquo;\";\n",
" expanded = !expanded;\n",
" });\n",
"})();\n",
"</script>\n"
],
"text/plain": [
"Figure(id='1752', ...)"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prof.visualize()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment