Skip to content

Instantly share code, notes, and snippets.

@rabernat

rabernat/etl_prefect.ipynb

Created September 9, 2019 15:58
Show Gist options
  • Save rabernat/df7ec5fbea982d48e0092927c4b1f474 to your computer and use it in GitHub Desktop.
Save rabernat/df7ec5fbea982d48e0092927c4b1f474 to your computer and use it in GitHub Desktop.
Download ZIP
Better prefect ETL
Display the source blob
Display the rendered blob
Raw
etl_prefect.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"# must be done before first numpy import\n",
"os.environ['NUMPY_EXPERIMENTAL_ARRAY_FUNCTION'] = '0'\n",
"\n",
"import xarray as xr\n",
"import numpy as np\n",
"import gcsfs\n",
"import zarr\n",
"import xrft\n",
"from pyresample import image, geometry\n",
"import json\n",
"from matplotlib import pyplot as plt\n",
"import dask\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.7/site-packages/distributed/dashboard/core.py:72: UserWarning: \n",
"Port 8787 is already in use. \n",
"Perhaps you already have a cluster running?\n",
"Hosting the diagnostics dashboard on a random port instead.\n",
" warnings.warn(\"\\n\" + msg)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6eec6c104a36453db38a7914f2e744d6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HTML(value='<h2>KubeCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n <style scoped>\\n .…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from dask_kubernetes import KubeCluster\n",
"cluster = KubeCluster(n_workers=4, env={\"NUMPY_EXPERIMENTAL_ARRAY_FUNCTION\": \"0\"})\n",
"cluster"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>tcp://10.32.60.163:40927</li>\n",
" <li><b>Dashboard: </b><a href='/user/0000-0001-5999-4917/proxy/44847/status' target='_blank'>/user/0000-0001-5999-4917/proxy/44847/status</a>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>4</li>\n",
" <li><b>Cores: </b>8</li>\n",
" <li><b>Memory: </b>46.00 GB</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: scheduler='tcp://10.32.60.163:40927' processes=4 cores=8>"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from dask.distributed import Client\n",
"\n",
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'tcp://10.32.168.4:37895': '0',\n",
" 'tcp://10.32.168.5:39727': '0',\n",
" 'tcp://10.32.2.4:40775': '0',\n",
" 'tcp://10.32.61.4:32927': '0'}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def check_env_vars():\n",
" import os\n",
" return os.environ['NUMPY_EXPERIMENTAL_ARRAY_FUNCTION']\n",
"\n",
"client.run(check_env_vars)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"from xmitgcm.llcreader.llcmodel import faces_dataset_to_latlon\n",
"\n",
"def setup_global_dataset(nt):\n",
"\n",
" gcs = gcsfs.GCSFileSystem(token='anon')\n",
" ds_sst_full = xr.open_zarr(gcs.get_mapper('pangeo-data/llc4320_surface/SST'),\n",
" consolidated=True, chunks=False)\n",
" ds_ssh_full = xr.open_zarr(gcs.get_mapper('pangeo-data/llc4320_surface/Eta'),\n",
" consolidated=True, chunks=False)\n",
" ds_grid = xr.open_zarr(gcs.get_mapper('pangeo-data/llc4320_surface/grid'),\n",
" consolidated=True)\n",
" coord_vars_to_keep = ['XC', 'YC']\n",
" rename_coords = {'XC': 'lon', 'YC': 'lat'}\n",
" ds_grid = (ds_grid.reset_coords()[coord_vars_to_keep]\n",
" .reset_coords(drop=True)\n",
" .set_coords(coord_vars_to_keep)\n",
" .rename(rename_coords))\n",
" \n",
" ds_full = xr.merge([ds_sst_full, ds_ssh_full, ds_grid])\n",
" \n",
" ds_faces = ds_full.isel(time=nt).chunk({'face': 1})\n",
" \n",
" # needed to make this work right, but import order matters\n",
" # too late if we call it here\n",
" #import os\n",
" #os.environ['NUMPY_EXPERIMENTAL_ARRAY_FUNCTION'] = '0'\n",
" \n",
" ds_ll = faces_dataset_to_latlon(ds_faces, metric_vector_pairs=[])\n",
" return ds_ll.drop('face')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 253 ms, sys: 32.7 ms, total: 286 ms\n",
"Wall time: 1.47 s\n"
]
},
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (i: 17280, j: 12960)\n",
"Coordinates:\n",
" * i (i) int64 0 1 2 3 4 5 6 ... 17274 17275 17276 17277 17278 17279\n",
" * j (j) int64 0 1 2 3 4 5 6 ... 12954 12955 12956 12957 12958 12959\n",
" time datetime64[ns] 2011-09-13\n",
" lon (j, i) float32 dask.array<shape=(12960, 17280), chunksize=(4320, 4320)>\n",
" lat (j, i) float32 dask.array<shape=(12960, 17280), chunksize=(4320, 4320)>\n",
"Data variables:\n",
" SST (j, i) float32 dask.array<shape=(12960, 17280), chunksize=(4320, 4320)>\n",
" Eta (j, i) float32 dask.array<shape=(12960, 17280), chunksize=(4320, 4320)>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time ds = setup_global_dataset(0)\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (point: 321)\n",
"Dimensions without coordinates: point\n",
"Data variables:\n",
" lon (point) float64 -32.52 -55.76 -166.8 ... -32.72 -12.24 0.4759\n",
" lat (point) float64 -74.01 -73.79 -73.56 -73.28 ... 62.82 68.88 69.47\n",
" j (point) int64 2190 2190 2190 2190 2190 ... 11370 11910 12450 12450\n",
" i (point) int64 30 10290 10830 11370 11910 ... 16230 30 1110 1650\n",
"Attributes:\n",
" region_size: 540"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def load_regions():\n",
" with open('land_free_regions_llc4320.json') as f:\n",
" d = json.load(f)\n",
" return xr.Dataset.from_dict(d)\n",
"\n",
"regions = load_regions()\n",
"regions"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def load_patch(ds=None, regions=None, nregion=None):\n",
" region = regions.sel(point=nregion)\n",
" size_j, size_i = regions.region_size, regions.region_size\n",
" i_start = region.i.values.item()\n",
" j_start = region.j.values.item()\n",
" ds_patch = ds.isel(i=slice(i_start, i_start + size_i),\n",
" j=slice(j_start, j_start + size_j))\n",
" return ds_patch"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def target_area_def(ds_patch):\n",
" \"\"\"Get a pyresample area_def for a given patch.\"\"\"\n",
" size_i = len(ds_patch.i)\n",
" size_j = len(ds_patch.j)\n",
" center_point = ds_patch.isel(i=size_i//2, j=size_j//2)\n",
" center_lon, center_lat = center_point.lon.values.item(), center_point.lat.values.item()\n",
" area_id = 'local_grid'\n",
" description = 'Local Lambert Equal Area Grid'\n",
" proj_id = 'local_grid'\n",
" proj_dict = {'proj': 'laea', 'lat_0': center_lat, 'lon_0': center_lon, 'units': 'm'} # 'a': 6371228.0,\n",
"\n",
" # 1 km resolution\n",
" width = 512\n",
" height = 512\n",
"\n",
" # 512 x 512 km\n",
" area_extent = (-256000., -256000., 256000., 256000.)\n",
" area_def = geometry.AreaDefinition(area_id, description, proj_id, proj_dict,\n",
" width, height, area_extent)\n",
" return area_def\n",
"\n",
"\n",
"def image_to_dataset(im):\n",
" \"\"\"Wrap pyresample image in xarray dataset.\"\"\"\n",
" geo_def = im.geo_def\n",
" units = geo_def.proj_dict['units']\n",
" lons, lats = geo_def.get_lonlats()\n",
" ds_resampled = xr.DataArray(im.image_data, dims=['y', 'x'],\n",
" coords={'x': ('x', geo_def.projection_x_coords, {'units': units}),\n",
" 'y': ('x', geo_def.projection_y_coords, {'units': units}),\n",
" 'lon': (['y', 'x'], lons),\n",
" 'lat': (['y', 'x'], lats)\n",
" }, name='SST').to_dataset()\n",
" ds_resampled.coords['projection'] = ('projection', [1], geo_def.proj_dict)\n",
" \n",
" # what metadata to save? this seems relevant\n",
" # http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#grid-mappings-and-projections\n",
" return ds_resampled\n",
"\n",
"def resample_patch(ds_patch):\n",
" \"\"\"Resample a patch of ocean.\"\"\"\n",
" grid_def = geometry.GridDefinition(lons=ds_patch.lon.values, lats=ds_patch.lat.values)\n",
" data = ds_patch.SST.to_masked_array()\n",
" im = image.ImageContainerBilinear(data, grid_def,\n",
" radius_of_influence=10000., fill_value=None)\n",
" \n",
" area_def = target_area_def(ds_patch)\n",
" im_resampled = im.resample(area_def)\n",
" #return image_to_dataset(im_resampled)\n",
" return im_resampled.image_data.filled(np.nan).astype('f4')"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
" -->\n",
"<!-- Title: %3 Pages: 1 -->\n",
"<svg width=\"344pt\" height=\"131pt\"\n",
" viewBox=\"0.00 0.00 344.00 131.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 127)\">\n",
"<title>%3</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-127 340,-127 340,4 -4,4\"/>\n",
"<!-- 140422754032496 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>140422754032496</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"168\" cy=\"-18\" rx=\"70.3881\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"168\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">do_etl_patch</text>\n",
"</g>\n",
"<!-- 140422752979824 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>140422752979824</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"27\" cy=\"-105\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"27\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n",
"</g>\n",
"<!-- 140422752979824&#45;&gt;140422754032496 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>140422752979824&#45;&gt;140422754032496</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M46.9028,-92.7195C69.0002,-79.085 105.1145,-56.8017 132.1645,-40.1113\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"134.2847,-42.9157 140.9572,-34.686 130.6089,-36.9585 134.2847,-42.9157\"/>\n",
"<text text-anchor=\"middle\" x=\"114\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nt</text>\n",
"</g>\n",
"<!-- 140423159254936 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>140423159254936</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"168\" cy=\"-105\" rx=\"96.3833\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"168\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">Constant[Dataset]</text>\n",
"</g>\n",
"<!-- 140423159254936&#45;&gt;140422754032496 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>140423159254936&#45;&gt;140422754032496</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M168,-86.9735C168,-75.1918 168,-59.5607 168,-46.1581\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.5001,-46.0033 168,-36.0034 164.5001,-46.0034 171.5001,-46.0033\"/>\n",
"<text text-anchor=\"middle\" x=\"195\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">regions</text>\n",
"</g>\n",
"<!-- 140422752979880 -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>140422752979880</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"309\" cy=\"-105\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"309\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10</text>\n",
"</g>\n",
"<!-- 140422752979880&#45;&gt;140422754032496 -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>140422752979880&#45;&gt;140422754032496</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M289.0972,-92.7195C266.9998,-79.085 230.8855,-56.8017 203.8355,-40.1113\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"205.3911,-36.9585 195.0428,-34.686 201.7153,-42.9157 205.3911,-36.9585\"/>\n",
"<text text-anchor=\"middle\" x=\"273.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nregion</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.dot.Digraph at 0x7fb6d0869240>"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from prefect import Flow, task, unmapped\n",
"\n",
"@task\n",
"def do_etl_patch(regions, nt, nregion):\n",
" # not necessary after fixing env variable issue\n",
" #with dask.config.set(scheduler='single-threaded'):\n",
" ds_full = setup_global_dataset(nt)\n",
" patch = load_patch(ds=ds, regions=regions, nregion=nregion)\n",
" patch.load()\n",
" patch_resampled = resample_patch(patch)\n",
"\n",
"@task\n",
"def combine_arrays(patches):\n",
" return np.stack(patches, axis=0)\n",
"\n",
" \n",
"with Flow('ETL_SST_simple') as flow_simple:\n",
" regions = load_regions()\n",
" do_etl_patch(regions, 0, 10)\n",
" \n",
"with Flow('ETL_SST_patches_mapped') as flow_mapped:\n",
" regions = load_regions()\n",
" nt = 0\n",
" nregion = list(range(regions.dims['point']))[:20] \n",
" patches = do_etl_patch.map(unmapped(regions), unmapped(nt), nregion)\n",
" result = combine_arrays(patches)\n",
"\n",
"flow_simple.visualize() "
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
" -->\n",
"<!-- Title: %3 Pages: 1 -->\n",
"<svg width=\"1431pt\" height=\"305pt\"\n",
" viewBox=\"0.00 0.00 1431.00 305.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 301)\">\n",
"<title>%3</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-301 1427,-301 1427,4 -4,4\"/>\n",
"<!-- 140422753793920 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>140422753793920</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"27\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"27\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2</text>\n",
"</g>\n",
"<!-- 140422753792968 -->\n",
"<g id=\"node15\" class=\"node\">\n",
"<title>140422753792968</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"711\" cy=\"-192\" rx=\"28.6953\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"711\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">List</text>\n",
"</g>\n",
"<!-- 140422753793920&#45;&gt;140422753792968 -->\n",
"<g id=\"edge15\" class=\"edge\">\n",
"<title>140422753793920&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M39.3297,-262.9648C49.5762,-250.9824 65.3017,-235.4407 83,-228 137.1002,-205.2553 543.1221,-195.325 671.9217,-192.7254\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.1844,-196.221 682.113,-192.5235 672.0456,-189.2224 672.1844,-196.221\"/>\n",
"<text text-anchor=\"middle\" x=\"103.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_3</text>\n",
"</g>\n",
"<!-- 140422753794816 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>140422753794816</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"99\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"99\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6</text>\n",
"</g>\n",
"<!-- 140422753794816&#45;&gt;140422753792968 -->\n",
"<g id=\"edge12\" class=\"edge\">\n",
"<title>140422753794816&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M110.0159,-262.5271C118.9762,-250.6445 132.7674,-235.4374 149,-228 196.2383,-206.3565 552.5635,-195.8341 672.0659,-192.8876\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.3116,-196.3827 682.2239,-192.6414 672.142,-189.3848 672.3116,-196.3827\"/>\n",
"<text text-anchor=\"middle\" x=\"169.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_7</text>\n",
"</g>\n",
"<!-- 140422753795712 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>140422753795712</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"171\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"171\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10</text>\n",
"</g>\n",
"<!-- 140422753795712&#45;&gt;140422753792968 -->\n",
"<g id=\"edge22\" class=\"edge\">\n",
"<title>140422753795712&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M180.2684,-261.6169C187.5107,-249.9343 198.7344,-235.3728 213,-228 253.3835,-207.1287 561.9476,-196.2904 672.0873,-193.0563\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.5179,-196.5454 682.4128,-192.7584 672.316,-189.5483 672.5179,-196.5454\"/>\n",
"<text text-anchor=\"middle\" x=\"238\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_11</text>\n",
"</g>\n",
"<!-- 140422753767944 -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>140422753767944</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"243\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"243\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14</text>\n",
"</g>\n",
"<!-- 140422753767944&#45;&gt;140422753792968 -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>140422753767944&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M252.6149,-261.6643C260.089,-250.0041 271.599,-235.4498 286,-228 319.8308,-210.4989 573.8175,-197.8749 672.2181,-193.5936\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.496,-197.085 682.3365,-193.159 672.1955,-190.0914 672.496,-197.085\"/>\n",
"<text text-anchor=\"middle\" x=\"311\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_15</text>\n",
"</g>\n",
"<!-- 140422753768840 -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>140422753768840</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"315\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"315\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18</text>\n",
"</g>\n",
"<!-- 140422753768840&#45;&gt;140422753792968 -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>140422753768840&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M324.9726,-261.736C332.6837,-250.1097 344.4818,-235.5663 359,-228 386.2912,-213.7768 586.5428,-199.7621 672.2521,-194.3446\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.5962,-197.83 682.358,-193.7122 672.1589,-190.8437 672.5962,-197.83\"/>\n",
"<text text-anchor=\"middle\" x=\"384\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_19</text>\n",
"</g>\n",
"<!-- 140423151543576 -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>140423151543576</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"854\" cy=\"-192\" rx=\"96.3833\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"854\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">Constant[Dataset]</text>\n",
"</g>\n",
"<!-- 140422753759184 -->\n",
"<g id=\"node18\" class=\"node\">\n",
"<title>140422753759184</title>\n",
"<polygon fill=\"none\" stroke=\"#000000\" points=\"937.5,-123 770.5,-123 770.5,-87 937.5,-87 937.5,-123\"/>\n",
"<text text-anchor=\"middle\" x=\"854\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">do_etl_patch &lt;map&gt;</text>\n",
"</g>\n",
"<!-- 140423151543576&#45;&gt;140422753759184 -->\n",
"<g id=\"edge19\" class=\"edge\">\n",
"<title>140423151543576&#45;&gt;140422753759184</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M854,-173.9735C854,-162.1918 854,-146.5607 854,-133.1581\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"857.5001,-133.0033 854,-123.0034 850.5001,-133.0034 857.5001,-133.0033\"/>\n",
"<text text-anchor=\"middle\" x=\"881\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">regions</text>\n",
"</g>\n",
"<!-- 140422753794592 -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>140422753794592</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"387\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"387\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5</text>\n",
"</g>\n",
"<!-- 140422753794592&#45;&gt;140422753792968 -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>140422753794592&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M397.3514,-261.8504C405.3095,-250.2779 417.3989,-235.7519 432,-228 472.7413,-206.37 605.261,-197.06 671.9364,-193.6653\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.2311,-197.1552 682.048,-193.1703 671.8888,-190.1636 672.2311,-197.1552\"/>\n",
"<text text-anchor=\"middle\" x=\"452.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_6</text>\n",
"</g>\n",
"<!-- 140422753795488 -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>140422753795488</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"459\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"459\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9</text>\n",
"</g>\n",
"<!-- 140422753795488&#45;&gt;140422753792968 -->\n",
"<g id=\"edge13\" class=\"edge\">\n",
"<title>140422753795488&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M466.1479,-261.5586C471.7461,-250.1622 480.6622,-235.9644 493,-228 521.9901,-209.286 617.8968,-199.0956 672.351,-194.691\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"672.6371,-198.1794 682.334,-193.9107 672.0916,-191.2006 672.6371,-198.1794\"/>\n",
"<text text-anchor=\"middle\" x=\"518\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_10</text>\n",
"</g>\n",
"<!-- 140422753792296 -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>140422753792296</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"531\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"531\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n",
"</g>\n",
"<!-- 140422753792296&#45;&gt;140422753792968 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>140422753792296&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M537.1942,-261.3672C542.0069,-250.1869 549.7719,-236.3129 561,-228 578.5101,-215.036 634.3531,-204.0091 672.7304,-197.6937\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.7256,-201.0789 683.0453,-196.0396 672.6172,-194.1672 673.7256,-201.0789\"/>\n",
"<text text-anchor=\"middle\" x=\"581.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_1</text>\n",
"</g>\n",
"<!-- 140422753767720 -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>140422753767720</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"603\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"603\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13</text>\n",
"</g>\n",
"<!-- 140422753767720&#45;&gt;140422753792968 -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>140422753767720&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M602.7676,-260.5653C603.5735,-249.9121 606.2896,-236.8996 614,-228 628.7912,-210.9275 652.4572,-201.941 672.7097,-197.216\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.5383,-200.6189 682.6231,-195.1677 672.1217,-193.7637 673.5383,-200.6189\"/>\n",
"<text text-anchor=\"middle\" x=\"639\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_14</text>\n",
"</g>\n",
"<!-- 140422753768616 -->\n",
"<g id=\"node11\" class=\"node\">\n",
"<title>140422753768616</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"675\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"675\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17</text>\n",
"</g>\n",
"<!-- 140422753768616&#45;&gt;140422753792968 -->\n",
"<g id=\"edge24\" class=\"edge\">\n",
"<title>140422753768616&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M670.085,-261.2998C668.1966,-251.1498 667.4466,-238.3998 672,-228 674.6193,-222.0175 678.7776,-216.5882 683.4219,-211.8644\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"685.8966,-214.3455 690.9689,-205.0438 681.2031,-209.1521 685.8966,-214.3455\"/>\n",
"<text text-anchor=\"middle\" x=\"697\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_18</text>\n",
"</g>\n",
"<!-- 140423151546040 -->\n",
"<g id=\"node12\" class=\"node\">\n",
"<title>140423151546040</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"995\" cy=\"-192\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"995\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n",
"</g>\n",
"<!-- 140423151546040&#45;&gt;140422753759184 -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>140423151546040&#45;&gt;140422753759184</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M975.0972,-179.7195C953.6136,-166.4637 918.881,-145.033 892.1101,-128.5148\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"893.7154,-125.3927 883.3672,-123.1202 890.0397,-131.3499 893.7154,-125.3927\"/>\n",
"<text text-anchor=\"middle\" x=\"940\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nt</text>\n",
"</g>\n",
"<!-- 140422753794368 -->\n",
"<g id=\"node13\" class=\"node\">\n",
"<title>140422753794368</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"747\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"747\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4</text>\n",
"</g>\n",
"<!-- 140422753794368&#45;&gt;140422753792968 -->\n",
"<g id=\"edge17\" class=\"edge\">\n",
"<title>140422753794368&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M739.8258,-261.5256C735.7764,-251.6755 730.6171,-239.1466 726,-228 724.7951,-225.0912 723.5331,-222.0514 722.2744,-219.0241\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"725.4138,-217.4585 718.3389,-209.572 718.9516,-220.1491 725.4138,-217.4585\"/>\n",
"<text text-anchor=\"middle\" x=\"751.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_5</text>\n",
"</g>\n",
"<!-- 140422753795264 -->\n",
"<g id=\"node14\" class=\"node\">\n",
"<title>140422753795264</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"819\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"819\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8</text>\n",
"</g>\n",
"<!-- 140422753795264&#45;&gt;140422753792968 -->\n",
"<g id=\"edge18\" class=\"edge\">\n",
"<title>140422753795264&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M807.5333,-262.4807C799.5424,-251.7705 788.1671,-238.0021 776,-228 766.2888,-220.0168 754.5518,-212.8947 743.7386,-207.135\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"745.0453,-203.8722 734.5481,-202.4368 741.859,-210.105 745.0453,-203.8722\"/>\n",
"<text text-anchor=\"middle\" x=\"810.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_9</text>\n",
"</g>\n",
"<!-- 140422753792968&#45;&gt;140422753759184 -->\n",
"<g id=\"edge16\" class=\"edge\">\n",
"<title>140422753792968&#45;&gt;140422753759184</title>\n",
"<path fill=\"none\" stroke=\"#000000\" stroke-dasharray=\"5,2\" d=\"M731.7934,-179.3495C753.6463,-166.0544 788.5189,-144.8382 815.4291,-128.4662\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"817.4957,-131.3058 824.2197,-123.1181 813.8574,-125.3256 817.4957,-131.3058\"/>\n",
"<text text-anchor=\"middle\" x=\"818.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nregion</text>\n",
"</g>\n",
"<!-- 140422753767496 -->\n",
"<g id=\"node16\" class=\"node\">\n",
"<title>140422753767496</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"891\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"891\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12</text>\n",
"</g>\n",
"<!-- 140422753767496&#45;&gt;140422753792968 -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>140422753767496&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M877.9045,-263.205C867.472,-251.6776 851.8609,-236.6181 835,-228 800.2284,-210.2271 786.0143,-222.4451 749,-210 746.8637,-209.2817 744.6899,-208.4749 742.5188,-207.6113\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.7098,-204.3132 733.1418,-203.5603 740.9337,-210.7392 743.7098,-204.3132\"/>\n",
"<text text-anchor=\"middle\" x=\"880\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_13</text>\n",
"</g>\n",
"<!-- 140422753768392 -->\n",
"<g id=\"node17\" class=\"node\">\n",
"<title>140422753768392</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"963\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"963\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16</text>\n",
"</g>\n",
"<!-- 140422753768392&#45;&gt;140422753792968 -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>140422753768392&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M950.7084,-262.7442C940.8246,-250.9754 925.8442,-235.8157 909,-228 844.0875,-197.8809 817.8279,-229.5834 749,-210 746.8322,-209.3832 744.6345,-208.6534 742.4458,-207.8467\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.6043,-204.538 733.0268,-203.9308 740.917,-211.0016 743.6043,-204.538\"/>\n",
"<text text-anchor=\"middle\" x=\"955\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_17</text>\n",
"</g>\n",
"<!-- 140423151545816 -->\n",
"<g id=\"node19\" class=\"node\">\n",
"<title>140423151545816</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"854\" cy=\"-18\" rx=\"85.2851\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"854\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">combine_arrays</text>\n",
"</g>\n",
"<!-- 140422753759184&#45;&gt;140423151545816 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>140422753759184&#45;&gt;140423151545816</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M854,-86.9735C854,-75.1918 854,-59.5607 854,-46.1581\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"857.5001,-46.0033 854,-36.0034 850.5001,-46.0034 857.5001,-46.0033\"/>\n",
"<text text-anchor=\"middle\" x=\"882.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">patches</text>\n",
"</g>\n",
"<!-- 140422753794144 -->\n",
"<g id=\"node20\" class=\"node\">\n",
"<title>140422753794144</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1035\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1035\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3</text>\n",
"</g>\n",
"<!-- 140422753794144&#45;&gt;140422753792968 -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>140422753794144&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1023.6637,-262.5813C1014.4717,-250.727 1000.3828,-235.5318 984,-228 936.4129,-206.1224 799.6228,-223.4348 749,-210 746.6782,-209.3838 744.3255,-208.6287 741.9881,-207.7804\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1628,-204.4787 732.5845,-203.8869 740.4849,-210.9462 743.1628,-204.4787\"/>\n",
"<text text-anchor=\"middle\" x=\"1024.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_4</text>\n",
"</g>\n",
"<!-- 140422753795040 -->\n",
"<g id=\"node21\" class=\"node\">\n",
"<title>140422753795040</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1107\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1107\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7</text>\n",
"</g>\n",
"<!-- 140422753795040&#45;&gt;140422753792968 -->\n",
"<g id=\"edge23\" class=\"edge\">\n",
"<title>140422753795040&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1094.1035,-262.9719C1083.4213,-250.9935 1067.1021,-235.4536 1049,-228 987.2437,-202.5716 813.6913,-226.5977 749,-210 746.6732,-209.403 744.3168,-208.6621 741.9767,-207.8241\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1469,-204.521 732.5673,-203.9529 740.4835,-210.9945 743.1469,-204.521\"/>\n",
"<text text-anchor=\"middle\" x=\"1092.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_8</text>\n",
"</g>\n",
"<!-- 140422753768168 -->\n",
"<g id=\"node22\" class=\"node\">\n",
"<title>140422753768168</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1179\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1179\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15</text>\n",
"</g>\n",
"<!-- 140422753768168&#45;&gt;140422753792968 -->\n",
"<g id=\"edge21\" class=\"edge\">\n",
"<title>140422753768168&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1165.4805,-263.3965C1153.9577,-251.317 1136.2421,-235.4327 1117,-228 1040.6243,-198.4982 828.4181,-229.909 749,-210 746.67,-209.4159 744.3111,-208.6845 741.9693,-207.8535\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1365,-204.5494 732.5561,-203.9971 740.4828,-211.0269 743.1365,-204.5494\"/>\n",
"<text text-anchor=\"middle\" x=\"1166\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_16</text>\n",
"</g>\n",
"<!-- 140422753769064 -->\n",
"<g id=\"node23\" class=\"node\">\n",
"<title>140422753769064</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1251\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1251\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19</text>\n",
"</g>\n",
"<!-- 140422753769064&#45;&gt;140422753792968 -->\n",
"<g id=\"edge14\" class=\"edge\">\n",
"<title>140422753769064&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1238.7048,-262.8817C1228.4774,-250.8534 1212.7606,-235.2906 1195,-228 1103.2385,-190.3325 845.3167,-233.7087 749,-210 746.6675,-209.4258 744.3068,-208.7019 741.9636,-207.8761\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1286,-204.5713 732.5476,-204.0313 740.4824,-211.0519 743.1286,-204.5713\"/>\n",
"<text text-anchor=\"middle\" x=\"1242\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_20</text>\n",
"</g>\n",
"<!-- 140422753795824 -->\n",
"<g id=\"node24\" class=\"node\">\n",
"<title>140422753795824</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1323\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1323\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11</text>\n",
"</g>\n",
"<!-- 140422753795824&#45;&gt;140422753792968 -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>140422753795824&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1311.4593,-262.3769C1302.0965,-250.4157 1287.7352,-235.176 1271,-228 1217.6622,-205.1291 805.3928,-223.706 749,-210 746.6658,-209.4327 744.3039,-208.7138 741.9598,-207.8917\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1233,-204.5865 732.5418,-204.0548 740.4822,-211.0691 743.1233,-204.5865\"/>\n",
"<text text-anchor=\"middle\" x=\"1317\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_12</text>\n",
"</g>\n",
"<!-- 140422753792632 -->\n",
"<g id=\"node25\" class=\"node\">\n",
"<title>140422753792632</title>\n",
"<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1396\" cy=\"-279\" rx=\"27\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1396\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1</text>\n",
"</g>\n",
"<!-- 140422753792632&#45;&gt;140422753792968 -->\n",
"<g id=\"edge20\" class=\"edge\">\n",
"<title>140422753792632&#45;&gt;140422753792968</title>\n",
"<path fill=\"none\" stroke=\"#000000\" d=\"M1385.055,-262.3688C1376.1319,-250.4033 1362.3561,-235.1618 1346,-228 1285.2089,-201.3814 813.5199,-225.5334 749,-210 746.6646,-209.4377 744.3017,-208.7226 741.957,-207.9032\"/>\n",
"<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1194,-204.5976 732.5376,-204.0722 740.4821,-211.0819 743.1194,-204.5976\"/>\n",
"<text text-anchor=\"middle\" x=\"1386.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">arg_2</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.dot.Digraph at 0x7fb6b85a0ba8>"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"flow_mapped.visualize()"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[2019-09-09 15:53:59,823] INFO - prefect.FlowRunner | Beginning Flow run for 'ETL_SST_simple'\n",
"[2019-09-09 15:53:59,828] INFO - prefect.FlowRunner | Starting flow run.\n",
"[2019-09-09 15:53:59,830] INFO - prefect.TaskRunner | Task '0': Starting task run...\n",
"[2019-09-09 15:53:59,832] INFO - prefect.TaskRunner | Task '0': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:53:59,833] INFO - prefect.TaskRunner | Task 'Constant[Dataset]': Starting task run...\n",
"[2019-09-09 15:53:59,836] INFO - prefect.TaskRunner | Task 'Constant[Dataset]': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:53:59,837] INFO - prefect.TaskRunner | Task '10': Starting task run...\n",
"[2019-09-09 15:53:59,839] INFO - prefect.TaskRunner | Task '10': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:53:59,840] INFO - prefect.TaskRunner | Task 'do_etl_patch': Starting task run...\n",
"[2019-09-09 15:54:05,517] INFO - prefect.TaskRunner | Task 'do_etl_patch': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:54:05,519] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 4.85 s, sys: 829 ms, total: 5.68 s\n",
"Wall time: 5.7 s\n"
]
},
{
"data": {
"text/plain": [
"<Success: \"All reference tasks succeeded.\">"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time flow_simple.run()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[2019-09-09 15:51:46,856] INFO - prefect.FlowRunner | Beginning Flow run for 'ETL_SST_patches_mapped'\n",
"[2019-09-09 15:51:46,858] INFO - prefect.FlowRunner | Starting flow run.\n",
"[2019-09-09 15:51:46,860] INFO - prefect.TaskRunner | Task 'Constant[Dataset]': Starting task run...\n",
"[2019-09-09 15:51:46,863] INFO - prefect.TaskRunner | Task 'Constant[Dataset]': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:46,865] INFO - prefect.TaskRunner | Task '0': Starting task run...\n",
"[2019-09-09 15:51:46,866] INFO - prefect.TaskRunner | Task '0': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:46,868] INFO - prefect.TaskRunner | Task '0': Starting task run...\n",
"[2019-09-09 15:51:46,870] INFO - prefect.TaskRunner | Task '0': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:46,871] INFO - prefect.TaskRunner | Task 'List': Starting task run...\n",
"[2019-09-09 15:51:46,873] INFO - prefect.TaskRunner | Task 'List': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:46,874] INFO - prefect.TaskRunner | Task 'do_etl_patch': Starting task run...\n",
"[2019-09-09 15:51:46,874] INFO - prefect.TaskRunner | Task 'do_etl_patch[0]': Starting task run...\n",
"[2019-09-09 15:51:52,977] INFO - prefect.TaskRunner | Task 'do_etl_patch[0]': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:52,978] INFO - prefect.TaskRunner | Task 'do_etl_patch': finished task run for task with final state: 'Mapped'\n",
"[2019-09-09 15:51:52,980] INFO - prefect.TaskRunner | Task 'combine_arrays': Starting task run...\n",
"[2019-09-09 15:51:52,983] INFO - prefect.TaskRunner | Task 'combine_arrays': finished task run for task with final state: 'Success'\n",
"[2019-09-09 15:51:52,984] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 5.28 s, sys: 1.16 s, total: 6.44 s\n",
"Wall time: 6.13 s\n"
]
},
{
"data": {
"text/plain": [
"<Success: \"All reference tasks succeeded.\">"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time flow_mapped.run()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"from prefect.engine.executors import DaskExecutor\n",
"executor = DaskExecutor(address=\"tcp://10.32.60.163:40927\")"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[2019-09-09 15:55:09,945] INFO - prefect.FlowRunner | Beginning Flow run for 'ETL_SST_simple'\n",
"[2019-09-09 15:55:09,948] INFO - prefect.FlowRunner | Starting flow run.\n",
"[2019-09-09 15:55:16,739] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 780 ms, sys: 51.4 ms, total: 832 ms\n",
"Wall time: 6.8 s\n"
]
},
{
"data": {
"text/plain": [
"<Success: \"All reference tasks succeeded.\">"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time flow_simple.run(executor=executor)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[2019-09-09 15:54:08,912] INFO - prefect.FlowRunner | Beginning Flow run for 'ETL_SST_patches_mapped'\n",
"[2019-09-09 15:54:08,914] INFO - prefect.FlowRunner | Starting flow run.\n",
"[2019-09-09 15:54:41,884] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 5.99 s, sys: 531 ms, total: 6.52 s\n",
"Wall time: 33 s\n"
]
},
{
"data": {
"text/plain": [
"<Success: \"All reference tasks succeeded.\">"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time flow_mapped.run(executor=executor)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment