Skip to content

Instantly share code, notes, and snippets.

@andersy005

andersy005/dynamic-catalog.ipynb

Last active January 11, 2019 21:26
Show Gist options
  • Save andersy005/9d6f8b861a6f72b9a666a9e92c0f45cb to your computer and use it in GitHub Desktop.
Save andersy005/9d6f8b861a6f72b9a666a9e92c0f45cb to your computer and use it in GitHub Desktop.
Download ZIP
Intake Plugins
Display the source blob
Display the rendered blob
Raw
dynamic-catalog.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Raw
environment.yml
name: intake-cmip-dev
channels:
- conda-forge
- intake
- defaults
dependencies:
- python=3.6
- numpy
- intake
- intake-xarray
- xarray
- netcdf4
- dask
- distributed
- pytest
- intake-cmip
Display the source blob
Display the rendered blob
Raw
intake-cmip5-prototype.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from collections import OrderedDict\n",
"\n",
"import intake_xarray.base\n",
"import numpy as np\n",
"import pandas as pd\n",
"import xarray as xr\n",
"\n",
"from intake_cmip.database import create_cmip5_database\n",
"import tempfile\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"CMIP5_TEST_DIR = tempfile.mkdtemp()\n",
"DB_DIR = tempfile.mkdtemp()\n",
"\n",
"\n",
"file_names = [\n",
" \"Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc\",\n",
" \"Tair_OImon_CSIRO-Mk3-6-0_historical_r2i1p1_200601-203512.nc\",\n",
"]\n",
"\n",
"\n",
"def setup():\n",
" test_paths = [\n",
" f\"{CMIP5_TEST_DIR}/output1/CCCma/CanESM2/rcp85/mon/atmos/Amon/r2i1p1\",\n",
" f\"{CMIP5_TEST_DIR}/output2/CSIRO-QCCCE/CSIRO-Mk3-6-0/historical/mon/seaIce/OImon/r2i1p1/v1/sic\",\n",
" ]\n",
"\n",
" ds = (\n",
" xr.tutorial.open_dataset(\"rasm\")\n",
" .load()\n",
" .isel(time=slice(0, 2), x=slice(0, 5), y=slice(0, 3))\n",
" )\n",
"\n",
" for idx, path in enumerate(test_paths):\n",
" os.makedirs(path, exist_ok=True)\n",
" file_path = f\"{path}/{file_names[idx]}\"\n",
" ds.to_netcdf(file_path, mode=\"w\")\n",
"\n",
"\n",
"def teardown():\n",
" try:\n",
" shutil.rmtree(CMIP5_TEST_DIR)\n",
" except BaseException:\n",
" pass\n",
"\n",
" try:\n",
" shutil.rmtree(DB_DIR)\n",
" except BaseException:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"**** Persisting CMIP5 database: /tmp/tmp661d5ea2/cmip5.csv ****\n"
]
}
],
"source": [
"setup()\n",
"create_cmip5_database(CMIP5_TEST_DIR, DB_DIR)\n",
"db_path = os.path.join(DB_DIR, 'cmip5.csv')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"class CMIP5DataSource(intake_xarray.base.DataSourceMixin):\n",
" \"\"\" Read CMIP5 data sets into xarray datasets\n",
"\n",
" \"\"\"\n",
"\n",
" container = \"xarray\"\n",
" version = \"0.0.1\"\n",
" partition_access = True\n",
" name = \"cmip5\"\n",
"\n",
" def __init__(\n",
" self,\n",
" database,\n",
" metadata=None,\n",
" ):\n",
"\n",
" # store important kwargs\n",
" self.database = self._read_database(database)\n",
" self.urlpath = \"\"\n",
" self.query = {}\n",
" self.query_results = None\n",
" self._ds = None\n",
" super(CMIP5DataSource, self).__init__(metadata=metadata)\n",
"\n",
" def _read_database(self, database):\n",
" if database == \"glade\":\n",
" database = glade_cmip5_db\n",
" if os.path.exists(database):\n",
" return pd.read_csv(database)\n",
" else:\n",
" raise FileNotFoundError(f\"{database}\")\n",
"\n",
" def _open_dataset(self):\n",
" \n",
" ens_filepaths = self._get_ens_filepaths()\n",
"\n",
" ds_list = [xr.open_mfdataset(paths) for paths in ens_filepaths.values()]\n",
" ens_list = list(ens_filepaths.keys())\n",
" self._ds = xr.concat(ds_list, dim=\"ensemble\")\n",
" self._ds[\"ensemble\"] = ens_list\n",
"\n",
" def to_xarray(self, dask=True):\n",
" \"\"\"Return dataset as an xarray instance\"\"\"\n",
" if dask:\n",
" return self.to_dask()\n",
" return self.read()\n",
" \n",
" def search(self, database='glade', model=None, experiment=None, frequency=None, realm=None, ensemble=None, varname=None):\n",
" self.query = {\n",
" \"model\": model,\n",
" \"experiment\": experiment,\n",
" \"frequency\": frequency,\n",
" \"realm\": realm,\n",
" \"ensemble\": ensemble,\n",
" \"varname\": varname,\n",
" }\n",
" database = self.database\n",
" condition = np.ones(len(database), dtype=bool)\n",
"\n",
" for key, val in self.query.items():\n",
" if val is not None:\n",
"\n",
" condition = condition & (database[key] == val)\n",
"\n",
" self.query_results = database.loc[condition]\n",
" return self\n",
" \n",
" \n",
" def results(self):\n",
" return self.query_results\n",
" \n",
" def _get_ens_filepaths(self):\n",
" if self.query_results.empty:\n",
" raise ValueError(\n",
" f\"No dataset found for:\\n \\\n",
" \\tmodel = {self.query['model']}\\n \\\n",
" \\texperiment = {self.query['experiment']} \\n \\\n",
" \\tfrequency = {self.query['frequency']} \\n \\\n",
" \\trealm = {self.query['realm']} \\n \\\n",
" \\tensemble = {self.query['ensemble']} \\n \\\n",
" \\tvarname = {self.query['varname']}\"\n",
" )\n",
" \n",
" \n",
" models = self.query_results.ensemble.nunique() > 1\n",
" experiments = self.query_results.experiment.nunique() > 1\n",
" frequencies = self.query_results.frequency.nunique() > 1\n",
" \n",
" if models or experiments or frequencies:\n",
" \n",
" raise ValueError(f\"Invalid results for search query = {self.query}.\\n\\\n",
" Please specify unique model, experiment, and frequency to use\")\n",
" \n",
" \n",
" # Check that the same varname is not in multiple realms\n",
" realm_list = self.query_results.realm.unique()\n",
" if len(realm_list) != 1:\n",
" raise ValueError(\n",
" f\"{self.query['varname']} found in multiple realms:\\\n",
" \\t{self.query['realm_list']}. Please specify the realm to use\"\n",
" )\n",
" \n",
" ds_dict = OrderedDict()\n",
" for ens in self.query_results[\"ensemble\"].unique():\n",
" ens_match = self.query_results[\"ensemble\"] == ens\n",
" paths = self.query_results.loc[ens_match][\"file_fullpath\"].tolist()\n",
" ds_dict[ens] = paths\n",
"\n",
" return ds_dict"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"source = CMIP5DataSource(database=db_path)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>ensemble</th>\n",
" <th>experiment</th>\n",
" <th>file_basename</th>\n",
" <th>file_fullpath</th>\n",
" <th>frequency</th>\n",
" <th>institution</th>\n",
" <th>model</th>\n",
" <th>realm</th>\n",
" <th>root</th>\n",
" <th>varname</th>\n",
" <th>version</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>r2i1p1</td>\n",
" <td>rcp85</td>\n",
" <td>Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>mon</td>\n",
" <td>CCCma</td>\n",
" <td>CanESM2</td>\n",
" <td>atmos</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>Tair</td>\n",
" <td>v0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" ensemble experiment file_basename \\\n",
"0 r2i1p1 rcp85 Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc \n",
"\n",
" file_fullpath frequency institution \\\n",
"0 /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... mon CCCma \n",
"\n",
" model realm root varname \\\n",
"0 CanESM2 atmos /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... Tair \n",
"\n",
" version \n",
"0 v0 "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.search(model='CanESM2', experiment='rcp85', realm='atmos').results()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'model': 'CanESM2',\n",
" 'experiment': 'rcp85',\n",
" 'frequency': None,\n",
" 'realm': 'atmos',\n",
" 'ensemble': None,\n",
" 'varname': None}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.query"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>ensemble</th>\n",
" <th>experiment</th>\n",
" <th>file_basename</th>\n",
" <th>file_fullpath</th>\n",
" <th>frequency</th>\n",
" <th>institution</th>\n",
" <th>model</th>\n",
" <th>realm</th>\n",
" <th>root</th>\n",
" <th>varname</th>\n",
" <th>version</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>r2i1p1</td>\n",
" <td>rcp85</td>\n",
" <td>Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>mon</td>\n",
" <td>CCCma</td>\n",
" <td>CanESM2</td>\n",
" <td>atmos</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>Tair</td>\n",
" <td>v0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>r2i1p1</td>\n",
" <td>historical</td>\n",
" <td>Tair_OImon_CSIRO-Mk3-6-0_historical_r2i1p1_200...</td>\n",
" <td>/tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3...</td>\n",
" <td>mon</td>\n",
" <td>CSIRO-QCCCE</td>\n",
" <td>CSIRO-Mk3-6-0</td>\n",
" <td>seaIce</td>\n",
" <td>/tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3...</td>\n",
" <td>Tair</td>\n",
" <td>v1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" ensemble experiment file_basename \\\n",
"0 r2i1p1 rcp85 Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc \n",
"1 r2i1p1 historical Tair_OImon_CSIRO-Mk3-6-0_historical_r2i1p1_200... \n",
"\n",
" file_fullpath frequency institution \\\n",
"0 /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... mon CCCma \n",
"1 /tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3... mon CSIRO-QCCCE \n",
"\n",
" model realm root \\\n",
"0 CanESM2 atmos /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... \n",
"1 CSIRO-Mk3-6-0 seaIce /tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3... \n",
"\n",
" varname version \n",
"0 Tair v0 \n",
"1 Tair v1 "
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.search(ensemble='r2i1p1').results()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'model': None,\n",
" 'experiment': None,\n",
" 'frequency': None,\n",
" 'realm': None,\n",
" 'ensemble': 'r2i1p1',\n",
" 'varname': None}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.query"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': 'r2i1p1', 'varname': None}.\n Please specify unique model, experiment, and frequency to use",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-10-ed68d1230ccc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_xarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36mto_xarray\u001b[0;34m(self, dask)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;34m\"\"\"Return dataset as an xarray instance\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;34m\"\"\"Return xarray object where variables are dask arrays\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m metadata = {\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0mens_filepaths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_ens_filepaths\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0mds_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_mfdataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpaths\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpaths\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mens_filepaths\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_get_ens_filepaths\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmodels\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mexperiments\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfrequencies\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Invalid results for search query = {self.query}.\\n Please specify unique model, experiment, and frequency to use\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': 'r2i1p1', 'varname': None}.\n Please specify unique model, experiment, and frequency to use"
]
}
],
"source": [
"source.to_xarray()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"source.query"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>ensemble</th>\n",
" <th>experiment</th>\n",
" <th>file_basename</th>\n",
" <th>file_fullpath</th>\n",
" <th>frequency</th>\n",
" <th>institution</th>\n",
" <th>model</th>\n",
" <th>realm</th>\n",
" <th>root</th>\n",
" <th>varname</th>\n",
" <th>version</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>r2i1p1</td>\n",
" <td>rcp85</td>\n",
" <td>Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>mon</td>\n",
" <td>CCCma</td>\n",
" <td>CanESM2</td>\n",
" <td>atmos</td>\n",
" <td>/tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m...</td>\n",
" <td>Tair</td>\n",
" <td>v0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>r2i1p1</td>\n",
" <td>historical</td>\n",
" <td>Tair_OImon_CSIRO-Mk3-6-0_historical_r2i1p1_200...</td>\n",
" <td>/tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3...</td>\n",
" <td>mon</td>\n",
" <td>CSIRO-QCCCE</td>\n",
" <td>CSIRO-Mk3-6-0</td>\n",
" <td>seaIce</td>\n",
" <td>/tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3...</td>\n",
" <td>Tair</td>\n",
" <td>v1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" ensemble experiment file_basename \\\n",
"0 r2i1p1 rcp85 Tair_Amon_CanESM2_rcp85_r2i1p1_200601-203512.nc \n",
"1 r2i1p1 historical Tair_OImon_CSIRO-Mk3-6-0_historical_r2i1p1_200... \n",
"\n",
" file_fullpath frequency institution \\\n",
"0 /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... mon CCCma \n",
"1 /tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3... mon CSIRO-QCCCE \n",
"\n",
" model realm root \\\n",
"0 CanESM2 atmos /tmp/tmpjdf3d5sv/output1/CCCma/CanESM2/rcp85/m... \n",
"1 CSIRO-Mk3-6-0 seaIce /tmp/tmpjdf3d5sv/output2/CSIRO-QCCCE/CSIRO-Mk3... \n",
"\n",
" varname version \n",
"0 Tair v0 \n",
"1 Tair v1 "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.search().results()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': 'r2i1p1', 'varname': None}.\n Please specify unique model, experiment, and frequency to use",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-12-37f78fce4ffc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mensemble\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'r2i1p1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_xarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36mto_xarray\u001b[0;34m(self, dask)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;34m\"\"\"Return dataset as an xarray instance\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;34m\"\"\"Return xarray object where variables are dask arrays\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m metadata = {\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0mens_filepaths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_ens_filepaths\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0mds_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_mfdataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpaths\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpaths\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mens_filepaths\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_get_ens_filepaths\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmodels\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mexperiments\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfrequencies\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Invalid results for search query = {self.query}.\\n Please specify unique model, experiment, and frequency to use\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': 'r2i1p1', 'varname': None}.\n Please specify unique model, experiment, and frequency to use"
]
}
],
"source": [
"source.search(ensemble='r2i1p1').to_xarray()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (ensemble: 1, time: 2, x: 5, y: 3)\n",
"Coordinates:\n",
" * time (time) object 1980-09-16 12:00:00 1980-10-17 00:00:00\n",
" xc (y, x) float64 189.2 189.4 189.6 189.7 ... 188.9 189.0 189.2 189.4\n",
" yc (y, x) float64 16.53 16.78 17.02 17.27 ... 17.1 17.34 17.59 17.84\n",
" * ensemble (ensemble) <U6 'r2i1p1'\n",
"Dimensions without coordinates: x, y\n",
"Data variables:\n",
" Tair (ensemble, time, y, x) float64 dask.array<shape=(1, 2, 3, 5), chunksize=(1, 2, 3, 5)>\n",
"Attributes:\n",
" title: /workspace/jhamman/processed/R1002RBRxaaa01a/l...\n",
" institution: U.W.\n",
" source: RACM R1002RBRxaaa01a\n",
" output_frequency: daily\n",
" output_mode: averaged\n",
" convention: CF-1.4\n",
" references: Based on the initial model of Liang et al., 19...\n",
" comment: Output from the Variable Infiltration Capacity...\n",
" nco_openmp_thread_number: 1\n",
" NCO: \"4.6.0\"\n",
" history: Tue Dec 27 14:15:22 2016: ncatted -a dimension..."
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"source.search(ensemble='r2i1p1', model='CanESM2').to_xarray()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': None, 'varname': None}.\n Please specify unique model, experiment, and frequency to use",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-13-60d16ce1c6e7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_xarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36mto_xarray\u001b[0;34m(self, dask)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;34m\"\"\"Return dataset as an xarray instance\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;34m\"\"\"Return xarray object where variables are dask arrays\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatashape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/srv/conda/lib/python3.6/site-packages/intake_xarray/base.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m metadata = {\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0mens_filepaths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_ens_filepaths\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0mds_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_mfdataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpaths\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpaths\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mens_filepaths\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-2d9157165608>\u001b[0m in \u001b[0;36m_get_ens_filepaths\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmodels\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mexperiments\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfrequencies\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Invalid results for search query = {self.query}.\\n Please specify unique model, experiment, and frequency to use\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: Invalid results for search query = {'model': None, 'experiment': None, 'frequency': None, 'realm': None, 'ensemble': None, 'varname': None}.\n Please specify unique model, experiment, and frequency to use"
]
}
],
"source": [
"source.search().to_xarray()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment