netcdfVersion.ipynb

xarrayVersion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "from IPython.display import HTML\n",
    "\n",
    "import pprint\n",
    "import sys\n",
    "sys.path.append(\"../ecgtools/\")\n",
    "import parsers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validating yaml file with yamale.\n",
      "Columns in dataframe:\n",
      "Index(['path', 'variable', 'time_range', 'experiment_name', 'member_id',\n",
      "       'ctrl_branch_year', 'model_name', 'time_freq', 'long_name', 'units',\n",
      "       'cell_methods'],\n",
      "      dtype='object')\n",
      "CPU times: user 11.1 s, sys: 32.1 ms, total: 11.1 s\n",
      "Wall time: 11.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "yaml_path = '/Users/mickelso/Desktop/xdev_work/test_output/testing_new.yaml'\n",
    "csv_path = '/Users/mickelso/Desktop/xdev_work/test_output/testing_new.csv'\n",
    "io_lib = 'xarray'\n",
    "\n",
    "Parser = parsers.YAML_Parser(yaml_path, csv_path, io_lib)\n",
    "df = Parser.parser()\n",
    "#HTML(Parser.parser().to_html(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validating yaml file with yamale.\n",
      "Columns in dataframe:\n",
      "Index(['path', 'variable', 'time_range', 'experiment_name', 'member_id',\n",
      "       'ctrl_branch_year', 'model_name', 'time_freq', 'long_name', 'units',\n",
      "       'cell_methods'],\n",
      "      dtype='object')\n",
      "CPU times: user 11.2 s, sys: 25.1 ms, total: 11.2 s\n",
      "Wall time: 11.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "yaml_path = '/Users/mickelso/Desktop/xdev_work/test_output/testing_new.yaml'\n",
    "csv_path = '/Users/mickelso/Desktop/xdev_work/test_output/testing_new.csv'\n",
    "io_lib = 'xarray'\n",
    "\n",
    "Parser = parsers.YAML_Parser(yaml_path, csv_path, io_lib)\n",
    "df = Parser.parser()\n",
    "#HTML(Parser.parser().to_html(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XARRAY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 232 ms, sys: 2.89 ms, total: 235 ms\n",
      "Wall time: 234 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "f = '/Users/mickelso/Desktop/xdev_work/data/CESM_DATA/b.e21.BWmaHIST.f19_g17.PMIP4-past1000.001/ocn/hist/b.e21.BWmaHIST.f19_g17.PMIP4-past1000.001.pop.h.0850-12.nc'\n",
    "# open file\n",
    "d = xr.open_dataset(f, decode_times=True, use_cftime=True, chunks={})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.9 ms, sys: 108 µs, total: 4.01 ms\n",
      "Wall time: 3.94 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# get time variable\n",
    "# find what the time (unlimited) dimension is\n",
    "if 'time' in d.coords:\n",
    "    times = d['time']\n",
    "    start = times[0].dt.strftime('%Y-%m-%d').data.item()\n",
    "    end = times[-1].dt.strftime('%Y-%m-%d').data.item()\n",
    "    date = start + \"-\" + end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.57 ms, sys: 75 µs, total: 1.64 ms\n",
      "Wall time: 1.77 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# got through variable list\n",
    "var_list = []    \n",
    "# loop through all variables\n",
    "for v in d.variables.keys():\n",
    "    # add all variables that are not coordinates to the catalog\n",
    "    if v not in d.coords:\n",
    "        var_list.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.17 ms, sys: 10 µs, total: 1.18 ms\n",
      "Wall time: 1.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#go through attr list\n",
    "attr_list = {}\n",
    "for v in var_list:\n",
    "    if hasattr(d.variables[v], 'units'):\n",
    "        attr_list[v] = getattr(d.variables[v], 'units')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.38 ms, sys: 30 µs, total: 1.41 ms\n",
      "Wall time: 1.43 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#close the file\n",
    "# close netcdf file\n",
    "d.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 241 ms, sys: 3.35 ms, total: 244 ms\n",
      "Wall time: 243 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "f = '/Users/mickelso/Desktop/xdev_work/data/CESM_DATA/b.e21.BWmaHIST.f19_g17.PMIP4-past1000.001/ocn/hist/b.e21.BWmaHIST.f19_g17.PMIP4-past1000.001.pop.h.0850-12.nc'\n",
    "# open file\n",
    "d = xr.open_dataset(f, decode_times=True, use_cftime=True, chunks={})\n",
    "\n",
    "# get time variable\n",
    "# find what the time (unlimited) dimension is\n",
    "if 'time' in d.coords:\n",
    "    times = d['time']\n",
    "    start = times[0].dt.strftime('%Y-%m-%d').data.item()\n",
    "    end = times[-1].dt.strftime('%Y-%m-%d').data.item()\n",
    "    date = start + \"-\" + end\n",
    "    \n",
    "# got through variable list\n",
    "var_list = []    \n",
    "# loop through all variables\n",
    "for v in d.variables.keys():\n",
    "    # add all variables that are not coordinates to the catalog\n",
    "    if v not in d.coords:\n",
    "        var_list.append(v)\n",
    "        \n",
    "#go through attr list\n",
    "attr_list = {}\n",
    "for v in var_list:\n",
    "    if hasattr(d.variables[v], 'units'):\n",
    "        attr_list[v] = getattr(d.variables[v], 'units')\n",
    "        \n",
    "#close the file\n",
    "# close netcdf file\n",
    "d.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}