Example Dataset for Tom (DataTree and CMIP6)

tom_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b4173e46",
   "metadata": {},
   "source": [
    "# Example datasets for tom to play with CMIP6 and DataTree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e3e39ac4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cmip6_preprocessing.utils import google_cmip_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e466835c",
   "metadata": {},
   "outputs": [],
   "source": [
    "col = google_cmip_col()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "02f92073",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CMCC-CM2-HR4',\n",
       " 'EC-Earth3P-HR',\n",
       " 'HadGEM3-GC31-MM',\n",
       " 'HadGEM3-GC31-HM',\n",
       " 'HadGEM3-GC31-LM',\n",
       " 'EC-Earth3P',\n",
       " 'ECMWF-IFS-HR',\n",
       " 'ECMWF-IFS-LR',\n",
       " 'HadGEM3-GC31-LL',\n",
       " 'CMCC-CM2-VHR4',\n",
       " 'GFDL-CM4',\n",
       " 'GFDL-AM4',\n",
       " 'IPSL-CM6A-LR',\n",
       " 'E3SM-1-0',\n",
       " 'CNRM-CM6-1',\n",
       " 'GFDL-ESM4',\n",
       " 'GFDL-OM4p5B',\n",
       " 'GFDL-CM4C192',\n",
       " 'GFDL-ESM2M',\n",
       " 'GISS-E2-1-G',\n",
       " 'GISS-E2-1-H',\n",
       " 'CNRM-ESM2-1',\n",
       " 'BCC-CSM2-MR',\n",
       " 'BCC-ESM1',\n",
       " 'MIROC6',\n",
       " 'AWI-CM-1-1-MR',\n",
       " 'EC-Earth3-LR',\n",
       " 'IPSL-CM6A-ATM-HR',\n",
       " 'CESM2',\n",
       " 'CESM2-WACCM',\n",
       " 'CNRM-CM6-1-HR',\n",
       " 'MRI-ESM2-0',\n",
       " 'CanESM5',\n",
       " 'SAM0-UNICON',\n",
       " 'GISS-E2-1-G-CC',\n",
       " 'UKESM1-0-LL',\n",
       " 'EC-Earth3',\n",
       " 'EC-Earth3-Veg',\n",
       " 'FGOALS-f3-L',\n",
       " 'CanESM5-CanOE',\n",
       " 'INM-CM4-8',\n",
       " 'INM-CM5-0',\n",
       " 'NESM3',\n",
       " 'MPI-ESM-1-2-HAM',\n",
       " 'CAMS-CSM1-0',\n",
       " 'MPI-ESM1-2-LR',\n",
       " 'MPI-ESM1-2-HR',\n",
       " 'MRI-AGCM3-2-S',\n",
       " 'MRI-AGCM3-2-H',\n",
       " 'MCM-UA-1-0',\n",
       " 'INM-CM5-H',\n",
       " 'KACE-1-0-G',\n",
       " 'NorESM2-LM',\n",
       " 'FGOALS-f3-H',\n",
       " 'FGOALS-g3',\n",
       " 'MIROC-ES2L',\n",
       " 'FIO-ESM-2-0',\n",
       " 'NorCPM1',\n",
       " 'NorESM1-F',\n",
       " 'MPI-ESM1-2-XR',\n",
       " 'CESM1-1-CAM5-CMIP5',\n",
       " 'E3SM-1-1',\n",
       " 'KIOST-ESM',\n",
       " 'NorESM2-MM',\n",
       " 'ACCESS-CM2',\n",
       " 'ACCESS-ESM1-5',\n",
       " 'CESM2-FV2',\n",
       " 'GISS-E2-2-G',\n",
       " 'CESM2-WACCM-FV2',\n",
       " 'GISS-E2-2-H',\n",
       " 'IITM-ESM',\n",
       " 'CIESM',\n",
       " 'E3SM-1-1-ECA',\n",
       " 'TaiESM1',\n",
       " 'AWI-ESM-1-1-LR',\n",
       " 'EC-Earth3-Veg-LR',\n",
       " 'CMCC-ESM2',\n",
       " 'CAS-ESM2-0',\n",
       " 'CMCC-CM2-SR5',\n",
       " 'EC-Earth3-AerChem',\n",
       " 'IPSL-CM5A2-INCA',\n",
       " 'BCC-CSM2-HR',\n",
       " 'EC-Earth3P-VHR',\n",
       " 'CESM1-WACCM-SC',\n",
       " 'EC-Earth3-CC',\n",
       " 'IPSL-CM6A-LR-INCA',\n",
       " 'MIROC-ES2H',\n",
       " 'ICON-ESM-LR',\n",
       " 'ssp585']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(col.df['source_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c4f0ba7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = col.search(\n",
    "    source_id=[\"CESM\", \"GFDL-ESM4\", \"MRI-ESM2-0\"],\n",
    "    variable_id=\"tos\",\n",
    "    experiment_id=[\"historical\", \"ssp585\"],\n",
    "    member_id=['r2i1p1f1', 'r1i1p1f1','r1i2p1f1'],\n",
    "    grid_label='gn'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "247cc908",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--> The keys in the returned dictionary of datasets are constructed as follows:\n",
      "\t'activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.zstore.dcpp_init_year.version'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='8' class='' max='8' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [8/8 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ddict = cat.to_dataset_dict(\n",
    "    aggregate=False, zarr_kwargs={\"use_cftime\": True, \"consolidated\": True}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "28cb186e",
   "metadata": {},
   "outputs": [],
   "source": [
    "size = []\n",
    "for ds in ddict.values():\n",
    "    size.append(ds.nbytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "66c743ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.822033656"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(size)/1e9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33f74b6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to preprocess the datasets use cmip6_preprocessing.preprocessing.combined_preprocessing"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}