Skip to content

Instantly share code, notes, and snippets.

@andersy005

andersy005/lens-on-aws.ipynb

Last active August 5, 2019 22:40
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save andersy005/2ba8f1813100d04bea4419c905ed457a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save andersy005/2ba8f1813100d04bea4419c905ed457a to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
lens-on-aws.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"import intake"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import s3fs\n",
"import xarray as xr\n",
"import os\n",
"from distributed.utils import format_bytes\n",
"from operator import mul\n",
"from functools import reduce\n",
"import yaml"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def print_ds_info(ds, var):\n",
" try:\n",
" dt = ds[var].dtype\n",
" itemsize = dt.itemsize\n",
" chunk_size = ds[var].data.chunksize\n",
" size = format_bytes(ds[var].nbytes)\n",
" _bytes = reduce(mul, chunk_size) * itemsize\n",
" chunk_size_bytes = format_bytes(_bytes)\n",
"\n",
" print(f\"Variable name: {var}\")\n",
" print(f\"Chunk shape: {chunk_size}\")\n",
" print(f\"Dataset shape: {ds[var].shape}\")\n",
" print(f\"Chunk size: {chunk_size_bytes}\")\n",
" print(f\"Dataset size: {size}\")\n",
" #print(f\"Chunks: {ds[var].chunks}\")\n",
" except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# fs = s3fs.S3FileSystem(anon=False, profile_name='intake-esm-tester')\n",
"# fs.ls('ncar-cesm-lens/atm')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"cdef = yaml.safe_load(\n",
" \"\"\"\n",
"name: AWS-CESM1-LE\n",
"collection_type: cesm-aws\n",
"data_sources:\n",
" land:\n",
" locations:\n",
" - name: monthly\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/lnd/monthly\n",
" file_extension: .zarr\n",
" - name: daily\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/lnd/monthly\n",
" file_extension: .zarr\n",
" ocean:\n",
" locations:\n",
" - name: monthly\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/ocn/monthly\n",
" file_extension: .zarr\n",
" ice:\n",
" locations:\n",
" - name: monthly-nh\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/ice_nh/monthly\n",
" file_extension: .zarr\n",
" \n",
" - name: monthly-sh\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/ice_sh/monthly\n",
" file_extension: .zarr\n",
" \n",
" - name: daily-nh\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/ice_nh/daily\n",
" file_extension: .zarr\n",
" \n",
" - name: daily-sh\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/ice_sh/daily\n",
" file_extension: .zarr\n",
" \n",
" atm:\n",
" locations:\n",
" - name: monthly\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/atm/monthly\n",
" file_extension: .zarr\n",
" \n",
" - name: daily\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/atm/daily\n",
" file_extension: .zarr\n",
" \n",
" - name: hourly6-1990-2005\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/atm/hourly6-1990-2005\n",
" file_extension: .zarr\n",
" \n",
" - name: hourly6-2026-2035\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/atm/hourly6-2026-2035\n",
" file_extension: .zarr\n",
" \n",
" - name: hourly6-2071-2080\n",
" loc_type: aws-s3\n",
" direct_access: True\n",
" urlpath: s3://ncar-cesm-lens/atm/hourly6-2071-2080\n",
" file_extension: .zarr\n",
" \n",
"\"\"\"\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"storage_options = {'anon': False, 'profile_name': 'intake-esm-tester'}"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"# mapper = get_mapper(url, **storage_options)\n",
"# xr.open_zarr(mapper, consolidated=True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Getting file listing: land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "cf4d18c2567a44ee93853da7278190a6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=14, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: land:daily:aws-s3:s3://ncar-cesm-lens/lnd/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "031e1c408a384ff09ae30e064f9c5cf0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=14, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: ocean:monthly:aws-s3:s3://ncar-cesm-lens/ocn/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93c4d3e0b29646aa9d0a763efedc5f7b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=6, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: ice:monthly-nh:aws-s3:s3://ncar-cesm-lens/ice_nh/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "69806be20f5841c1ab42c5967eb6d698",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=4, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: ice:monthly-sh:aws-s3:s3://ncar-cesm-lens/ice_sh/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "acb985f260884f17b187a4d9f9f6abe8",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=4, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: ice:daily-nh:aws-s3:s3://ncar-cesm-lens/ice_nh/daily\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a237fbfc81db48e7b050d3dedad97086",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=4, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: ice:daily-sh:aws-s3:s3://ncar-cesm-lens/ice_sh/daily\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d074a511c26c4e8c931de66489a51bac",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=4, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: atm:monthly:aws-s3:s3://ncar-cesm-lens/atm/monthly\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f7defd0bd7c0453f8b3dab658e092c46",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=48, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: atm:daily:aws-s3:s3://ncar-cesm-lens/atm/daily\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d71eff8f15b045d58adc219a9590a752",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=48, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: atm:hourly6-1990-2005:aws-s3:s3://ncar-cesm-lens/atm/hourly6-1990-2005\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2b3daf5eea1d445e81e37c23344bb949",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=11, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: atm:hourly6-2026-2035:aws-s3:s3://ncar-cesm-lens/atm/hourly6-2026-2035\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5347e4dea29249398018ddefb109e130",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=11, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Getting file listing: atm:hourly6-2071-2080:aws-s3:s3://ncar-cesm-lens/atm/hourly6-2071-2080\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b8e3faa054c04e3a99992de289c84a39",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='file listing', max=11, style=ProgressStyle(description_width=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 179 entries, 0 to 178\n",
"Data columns (total 9 columns):\n",
"resource 179 non-null object\n",
"resource_type 179 non-null object\n",
"direct_access 179 non-null bool\n",
"experiment 179 non-null object\n",
"frequency 179 non-null object\n",
"component 179 non-null object\n",
"variable 179 non-null object\n",
"store_fullpath 179 non-null object\n",
"store_bucketname 179 non-null object\n",
"dtypes: bool(1), object(8)\n",
"memory usage: 11.5+ KB\n",
"None\n",
"Persisting AWS-CESM1-LE at : /home/jovyan/.intake_esm/collections/cesm-aws/AWS-CESM1-LE.cesm-aws.csv\n"
]
}
],
"source": [
"col = intake.open_esm_metadatastore(collection_input_definition=cdef, overwrite_existing=True, storage_options=storage_options)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>resource</th>\n",
" <th>resource_type</th>\n",
" <th>direct_access</th>\n",
" <th>experiment</th>\n",
" <th>frequency</th>\n",
" <th>component</th>\n",
" <th>variable</th>\n",
" <th>store_fullpath</th>\n",
" <th>store_bucketname</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo...</td>\n",
" <td>aws-s3</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>monthly</td>\n",
" <td>lnd</td>\n",
" <td>FSNO</td>\n",
" <td>s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-FSN...</td>\n",
" <td>s3://ncar-cesm-lens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo...</td>\n",
" <td>aws-s3</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>monthly</td>\n",
" <td>lnd</td>\n",
" <td>H2OSNO</td>\n",
" <td>s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-H2O...</td>\n",
" <td>s3://ncar-cesm-lens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo...</td>\n",
" <td>aws-s3</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>monthly</td>\n",
" <td>lnd</td>\n",
" <td>QRUNOFF</td>\n",
" <td>s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-QRU...</td>\n",
" <td>s3://ncar-cesm-lens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo...</td>\n",
" <td>aws-s3</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>monthly</td>\n",
" <td>lnd</td>\n",
" <td>RAIN</td>\n",
" <td>s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-RAI...</td>\n",
" <td>s3://ncar-cesm-lens</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo...</td>\n",
" <td>aws-s3</td>\n",
" <td>True</td>\n",
" <td>20C</td>\n",
" <td>monthly</td>\n",
" <td>lnd</td>\n",
" <td>SNOW</td>\n",
" <td>s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-SNO...</td>\n",
" <td>s3://ncar-cesm-lens</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" resource resource_type \\\n",
"0 land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo... aws-s3 \n",
"1 land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo... aws-s3 \n",
"2 land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo... aws-s3 \n",
"3 land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo... aws-s3 \n",
"4 land:monthly:aws-s3:s3://ncar-cesm-lens/lnd/mo... aws-s3 \n",
"\n",
" direct_access experiment frequency component variable \\\n",
"0 True 20C monthly lnd FSNO \n",
"1 True 20C monthly lnd H2OSNO \n",
"2 True 20C monthly lnd QRUNOFF \n",
"3 True 20C monthly lnd RAIN \n",
"4 True 20C monthly lnd SNOW \n",
"\n",
" store_fullpath store_bucketname \n",
"0 s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-FSN... s3://ncar-cesm-lens \n",
"1 s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-H2O... s3://ncar-cesm-lens \n",
"2 s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-QRU... s3://ncar-cesm-lens \n",
"3 s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-RAI... s3://ncar-cesm-lens \n",
"4 s3://ncar-cesm-lens/lnd/monthly/cesmLE-20C-SNO... s3://ncar-cesm-lens "
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.df.head()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(array(['FSNO', 'H2OSNO', 'QRUNOFF', 'RAIN', 'SNOW', 'SOILLIQ',\n",
" 'SOILWATER_10CM', 'SALT', 'SSH', 'SST', 'aice', 'hi', 'aice_d',\n",
" 'hi_d', 'FLNS', 'FLNSC', 'FLUT', 'FSNS', 'FSNSC', 'FSNTOA',\n",
" 'ICEFRAC', 'LHFLX', 'PRECC', 'PRECL', 'PRECSC', 'PRECSL', 'PSL',\n",
" 'Q', 'SHFLX', 'T', 'TMQ', 'TREFHT', 'TREFHTMN', 'TREFHTMX', 'TS',\n",
" 'U', 'V', 'Z3', 'PRECT', 'PRECTMX', 'Q850', 'UBOT', 'WSPDSRFAV',\n",
" 'Z500', 'PS', 'QREFHT'], dtype=object),\n",
" array(['20C', 'RCP85'], dtype=object),\n",
" array(['lnd', 'ocn', 'ice_nh', 'ice_sh', 'atm'], dtype=object))"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.df.variable.unique(), col.df.experiment.unique(), col.df.component.unique()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "fc2421dc01ff47a2a671ada5f33f2d07",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='dataset(s)', max=1, style=ProgressStyle(description_width='in…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3644f45f0ec9482b89c1db12047a960b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, description='variable(s)', max=11, style=ProgressStyle(description_width='…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (ilev: 31, lat: 192, lev: 30, lon: 288, member_id: 40, nbnd: 2, slat: 191, slon: 288, time: 23360)\n",
"Coordinates:\n",
" * member_id (member_id) int64 1 2 3 4 5 6 7 ... 34 35 101 102 103 104 105\n",
" * ilev (ilev) float64 2.255 5.032 10.16 18.56 ... 967.5 985.1 1e+03\n",
" * lat (lat) float64 -90.0 -89.06 -88.12 -87.17 ... 88.12 89.06 90.0\n",
" * lev (lev) float64 3.643 7.595 14.36 24.61 ... 957.5 976.3 992.6\n",
" * lon (lon) float64 0.0 1.25 2.5 3.75 ... 355.0 356.2 357.5 358.8\n",
" * slat (slat) float64 -89.53 -88.59 -87.64 ... 87.64 88.59 89.53\n",
" * slon (slon) float64 -0.625 0.625 1.875 3.125 ... 355.6 356.9 358.1\n",
" * time (time) object 1990-01-01 06:00:00 ... 2006-01-01 00:00:00\n",
"Dimensions without coordinates: nbnd\n",
"Data variables:\n",
" P0 float64 ...\n",
" PRECT (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" ch4vmr (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" co2vmr (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" date (time) int32 dask.array<shape=(23360,), chunksize=(504,)>\n",
" date_written (time) |S8 dask.array<shape=(23360,), chunksize=(504,)>\n",
" datesec (time) int32 dask.array<shape=(23360,), chunksize=(504,)>\n",
" f11vmr (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" f12vmr (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" gw (lat) float64 dask.array<shape=(192,), chunksize=(192,)>\n",
" hyai (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hyam (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" hybi (ilev) float64 dask.array<shape=(31,), chunksize=(31,)>\n",
" hybm (lev) float64 dask.array<shape=(30,), chunksize=(30,)>\n",
" mdt int32 ...\n",
" n2ovmr (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" nbdate int32 ...\n",
" nbsec int32 ...\n",
" ndbase int32 ...\n",
" ndcur (time) int32 dask.array<shape=(23360,), chunksize=(504,)>\n",
" nlon (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" nsbase int32 ...\n",
" nscur (time) int32 dask.array<shape=(23360,), chunksize=(504,)>\n",
" nsteph (time) int32 dask.array<shape=(23360,), chunksize=(504,)>\n",
" ntrk int32 ...\n",
" ntrm int32 ...\n",
" ntrn int32 ...\n",
" sol_tsi (time) float64 dask.array<shape=(23360,), chunksize=(504,)>\n",
" time_bnds (time, nbnd) object dask.array<shape=(23360, 2), chunksize=(11680, 2)>\n",
" time_written (time) |S8 dask.array<shape=(23360,), chunksize=(504,)>\n",
" w_stag (slat) float64 dask.array<shape=(191,), chunksize=(191,)>\n",
" wnummax (lat) int32 dask.array<shape=(192,), chunksize=(192,)>\n",
" PS (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" PSL (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" Q (member_id, time, lev, lat, lon) float32 dask.array<shape=(40, 23360, 30, 192, 288), chunksize=(2, 504, 1, 192, 288)>\n",
" QREFHT (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" T (member_id, time, lev, lat, lon) float32 dask.array<shape=(40, 23360, 30, 192, 288), chunksize=(2, 504, 1, 192, 288)>\n",
" TREFHT (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" TS (member_id, time, lat, lon) float32 dask.array<shape=(40, 23360, 192, 288), chunksize=(2, 504, 192, 288)>\n",
" U (member_id, time, lev, lat, lon) float32 dask.array<shape=(40, 23360, 30, 192, 288), chunksize=(2, 504, 1, 192, 288)>\n",
" V (member_id, time, lev, lat, lon) float32 dask.array<shape=(40, 23360, 30, 192, 288), chunksize=(2, 504, 1, 192, 288)>\n",
" Z3 (member_id, time, lev, lat, lon) float32 dask.array<shape=(40, 23360, 30, 192, 288), chunksize=(2, 504, 1, 192, 288)>\n",
"Attributes:\n",
" important_note: This data is part of the project 'Blind Evalua...\n",
" NCO: 4.3.4\n",
" initial_file: b.e11.B20TRC5CNBDRD.f09_g16.001.cam.i.1920-01-...\n",
" Conventions: CF-1.0\n",
" topography_file: /scratch/p/pjk/mudryk/cesm1_1_2_LENS/inputdata...\n",
" nco_openmp_thread_number: 1\n",
" history: 2019-08-01 00:15:18.487461 xarray.open_dataset...\n",
" source: CAM\n",
" revision_Id: $Id$\n",
" title: UNSET\n",
" logname: mudryk\n",
" Version: $Name$"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"_, ds = col.search(component='atm', frequency=['hourly6-1990-2005']).to_xarray(consolidated=True).popitem()\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total Dataset Size: 32.24 TB\n",
"------------------------------------------------------------\n",
"Variable name: PS\n",
"Chunk shape: (6, 504, 192, 288)\n",
"Dataset shape: (40, 23360, 192, 288)\n",
"Chunk size: 668.86 MB\n",
"Dataset size: 206.67 GB\n",
"************************************************************\n",
"Variable name: PSL\n",
"Chunk shape: (6, 504, 192, 288)\n",
"Dataset shape: (40, 23360, 192, 288)\n",
"Chunk size: 668.86 MB\n",
"Dataset size: 206.67 GB\n",
"************************************************************\n",
"Variable name: Q\n",
"Chunk shape: (2, 504, 1, 192, 288)\n",
"Dataset shape: (40, 23360, 30, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 6.20 TB\n",
"************************************************************\n",
"Variable name: QREFHT\n",
"Chunk shape: (2, 504, 192, 288)\n",
"Dataset shape: (40, 23360, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 206.67 GB\n",
"************************************************************\n",
"Variable name: T\n",
"Chunk shape: (2, 504, 1, 192, 288)\n",
"Dataset shape: (40, 23360, 30, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 6.20 TB\n",
"************************************************************\n",
"Variable name: TREFHT\n",
"Chunk shape: (2, 504, 192, 288)\n",
"Dataset shape: (40, 23360, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 206.67 GB\n",
"************************************************************\n",
"Variable name: TS\n",
"Chunk shape: (2, 504, 192, 288)\n",
"Dataset shape: (40, 23360, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 206.67 GB\n",
"************************************************************\n",
"Variable name: U\n",
"Chunk shape: (2, 504, 1, 192, 288)\n",
"Dataset shape: (40, 23360, 30, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 6.20 TB\n",
"************************************************************\n",
"Variable name: V\n",
"Chunk shape: (2, 504, 1, 192, 288)\n",
"Dataset shape: (40, 23360, 30, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 6.20 TB\n",
"************************************************************\n",
"Variable name: Z3\n",
"Chunk shape: (2, 504, 1, 192, 288)\n",
"Dataset shape: (40, 23360, 30, 192, 288)\n",
"Chunk size: 222.95 MB\n",
"Dataset size: 6.20 TB\n",
"************************************************************\n"
]
}
],
"source": [
"print(f'Total Dataset Size: {format_bytes(ds.nbytes)}')\n",
"print(\"-\"*60)\n",
"for variable in ['PS', 'PSL', 'Q', 'QREFHT', 'T', 'TREFHT', 'TS', 'U', 'V', 'Z3']:\n",
" print_ds_info(ds, variable)\n",
" print(\"*\"* 60)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:dev]",
"language": "python",
"name": "conda-env-dev-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment