Make CF-compliant netcdf MVP files

MVPTranslate.ipynb

{
  "cells": [
    {
      "metadata": {
        "collapsed": true
      },
      "cell_type": "markdown",
      "source": "## TODO:\n  - Check T/S lagging...\n  - Figure out *where* in cross sections the stagnation is.\n  - make regular cross-section plots T,S,U,V for each survey.\n\n### Use Headings Map to get list of headings\n "
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "## Prelim"
    },
    {
      "metadata": {
        "collapsed": false,
        "trusted": true
      },
      "cell_type": "code",
      "source": "import pickle\nimport matplotlib.pyplot as plt\nimport numpy as np\n%matplotlib nbagg\nimport seawater\nplt.style.use('ggplot')\nimport xarray as xr\n#from mvpLA16 import *\n# from jmkdata import *\nimport matplotlib.dates as mdates\nimport datetime\n%load_ext autoreload\n%autoreload 2\n",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": "/Users/jklymak/anaconda3/lib/python3.6/site-packages/matplotlib/style/core.py:197: UserWarning: In /Users/jklymak/.matplotlib/stylelib/ggplotjmk.mplstyle: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.\n  warnings.warn(message)\n/Users/jklymak/anaconda3/lib/python3.6/site-packages/xarray/core/formatting.py:16: FutureWarning: The pandas.tslib module is deprecated and will be removed in a future version.\n  from pandas.tslib import OutOfBoundsDatetime\n",
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "## CF compliance:\n\n  - [Standard Names](http://cfconventions.org/Data/cf-standard-names/44/build/cf-standard-name-table.html)\n  - [CF Conventions](http://cfconventions.org/)\n"
    },
    {
      "metadata": {
        "trusted": true,
        "collapsed": true
      },
      "cell_type": "code",
      "source": "sums = ['Survey A of Laser 2016 MVP Survey; Large-scale survey before first array of floats was deployed',\n'Survey B of Laser 2016 MVP Survey; Cloverleaf pattern while drifters deployed',\n'Survey C of Laser 2016 MVP Survey; Run onshore to NW',\n'Survey D of Laser 2016 MVP Survey; Survey of messy area on continental slope break',\n'Survey E of Laser 2016 MVP Survey; Frontal survey tracking floats in front that runs to south',\n'Survey F of Laser 2016 MVP Survey; Frontal survey tracking floats in front that runs to south',\n'Survey G of Laser 2016 MVP Survey; Frontal survey tracking floats; stayed in one location and let front advect past',\n'Survey H of Laser 2016 MVP Survey; Frontal survey tracking floats; stayed in one location and let front advect past', \n'Survey I of Laser 2016 MVP Survey; Vortex to north near continental slope']\ntds = ['A','B','C','D','E','F','G','H','I']",
      "execution_count": 2,
      "outputs": []
    },
    {
      "metadata": {
        "collapsed": false,
        "scrolled": false,
        "trusted": true
      },
      "cell_type": "code",
      "source": "# I just edited these by hand.  Prob not very good for reproducibility!\nfor nn,td in enumerate(tds):\n    summary = sums[nn]\n\n    cgrid = pickle.load(open('./pickleGrid/survey'+td+'.pickle','rb' ), encoding='latin1')\n    print(cgrid.keys())\n    # OK, make an xarray data set\n\n    # get time:\n    time = cgrid['matlabtime'] - mdates.date2num(datetime.date(2017,1,1))\n\n    cgridx = xr.Dataset(\n            coords={'longitude': (['cast_number'], cgrid['longitude'],\n                                  {'units':'degrees_east', 'long_name':'Lon [degrees]','standard_name':'longitude'}),\n                    'latitude': (['cast_number'], cgrid['latitude'], \n                                 {'units':'degrees_north', 'long_name':'Lat [degrees]', 'standard_name':'latitude',}),\n                    'depths': (['depths'],cgrid['depths'], \n                               {'units':'meters','positive':'down','standard_name':'depth'}),\n                    'cast_number': (['cast_number'],cgrid['cast_number'], \n                                   {'cf_role':'profile_id', 'long_name':'Cast Number'}), \n                    'time': ('cast_number', time, {'units': 'days since 2016-01-01','timezone':'UTC', 'standard_name':'time'})\n                }\n            )\n    #cgrid['cast_number'].attrs['cf_role'] = 'profile_id'\n\n    cgridx['temp'] = (['depths','cast_number'], cgrid['temp'])\n    cgridx['temp'].attrs['units'] = 'C'\n    cgridx['temp'].attrs['standard_name'] = 'sea_water_temperature'\n    cgridx['temp'].attrs['long_name'] = 'T [^oC]'\n\n    cgridx['cond'] = (['depths','cast_number'], cgrid['cond'])\n    cgridx['cond'].attrs['units'] = 'mS cm-1'\n    cgridx['cond'].attrs['standard_name'] = 'sea_water_electrical_conductivity'\n    cgridx['cond'].attrs['long_name'] = 'C [mS\\,cm^{-1}]'\n\n    cgridx['salinity'] = (['depths','cast_number'], cgrid['salinity'])\n    cgridx['salinity'].attrs['units'] = '1'\n    cgridx['salinity'].attrs['standard_name'] = 'sea_water_practical_salinity'\n    cgridx['salinity'].attrs['long_name'] = 'S [psu]'\n\n    cgridx['pden'] = (['depths','cast_number'], cgrid['pden']-1000.)\n    cgridx['pden'].attrs['units'] = 'kg m^-3'\n    cgridx['pden'].attrs['standard_name'] = 'sea_water_sigma_theta'\n    cgridx['pden'].attrs['long_name'] = '\\sigma_{\\theta} [kg\\,m^{-3}]'\n\n    cgridx['pressure'] = (['depths','cast_number'], cgrid['pressure'])\n    cgridx['pressure'].attrs['units'] = 'dbar'\n    cgridx['pressure'].attrs['standard_name'] = 'sea_water_pressure'\n    cgridx['pressure'].attrs['long_name'] = 'P [dbar]'\n\n    cgridx['analog'] = (['depths','cast_number'], cgrid['analog'])\n    cgridx['analog'].attrs['units'] = 'V'\n    # cgridx['analog'].attrs['standard_name'] = ''\n    cgridx['analog'].attrs['long_name'] = 'Analog [V]'\n\n    cgridx.attrs['title'] = 'LA16: Survey %s'%td\n    cgridx.attrs['summary'] = summary\n    cgridx.attrs['history'] = 'Version to GRIIDC, June/July 2017'\n    cgridx.attrs['creator_name'] = 'Jody Klymak'\n    cgridx.attrs['creator_email'] = 'jklymak@uvic.ca'\n    cgridx.attrs['institution'] = 'U. of Victoria'\n    cgridx.attrs['source'] = 'UVic Moving Vessel Profiler'\n\n    cf = xr.decode_cf(cgridx)\n\n    cgridx.attrs['processing_level'] = 'medium: first pass and some science has been done with the CTD data.  Definite salinity spikes still exist.'\n    cgridx.attrs['time_coverage_start'] = str(cf.time[0].values)\n    cgridx.attrs['time_coverage_end'] = str(cf.time[-1].values)\n    cgridx.attrs['featureType'] = 'Profile'\n    cgridx.attrs['comment'] = 'Made with MVP_dataprocess.py https://github.com/jklymak/pythonMvp and a script to create the netcdf file'\n    cgridx.attrs['Conventions'] = 'CF-1.6'\n    cgridx.attrs['references'] = 'Cruise report: http://web.uvic.ca/~jklymak/LA16'\n\n    cgridx.to_netcdf('netcdf/Survey%s.nc'%td,'w')\n\nprint(cgridx)\nprint(cgridx.temp)\n\n",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": "dict_keys(['cast_number', 'pden', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\ndict_keys(['cast_number', 'pden', 'alongx', 'temp', 'density', 'bottom', 'longitude', 'salinity', 'depths', 'pressure', 'cond', 'mtime', 'latitude', 'matlabtime', 'analog'])\n<xarray.Dataset>\nDimensions:      (cast_number: 779, depths: 321)\nCoordinates:\n    longitude    (cast_number) float64 -88.7 -88.7 -88.69 -88.68 -88.67 ...\n    latitude     (cast_number) float64 28.83 28.83 28.83 28.83 28.83 28.82 ...\n  * depths       (depths) float64 0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 ...\n  * cast_number  (cast_number) int64 1926 1927 1928 1929 1930 1931 1932 1933 ...\n    time         (cast_number) float64 36.17 36.17 36.17 36.18 36.18 36.18 ...\nData variables:\n    temp         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    cond         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    salinity     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pden         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pressure     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    analog       (depths, cast_number) float64 nan nan nan nan nan nan nan ...\nAttributes:\n    title:                LA16: Survey I\n    summary:              Survey I of Laser 2016 MVP Survey; Vortex to north ...\n    history:              Version to GRIIDC, June/July 2017\n    creator_name:         Jody Klymak\n    creator_email:        jklymak@uvic.ca\n    institution:          U. of Victoria\n    source:               UVic Moving Vessel Profiler\n    processing_level:     medium: first pass and some science has been done w...\n    time_coverage_start:  2016-02-06T04:04:01.603200000\n    time_coverage_end:    2016-02-12T12:42:07.286400000\n    featureType:          Profile\n    comment:              Made with MVP_dataprocess.py https://github.com/jkl...\n    Conventions:          CF-1.6\n    references:           Cruise report: http://web.uvic.ca/~jklymak/LA16\n<xarray.DataArray 'temp' (depths: 321, cast_number: 779)>\narray([[ nan,  nan,  nan, ...,  nan,  nan,  nan],\n       [ nan,  nan,  nan, ...,  nan,  nan,  nan],\n       [ nan,  nan,  nan, ...,  nan,  nan,  nan],\n       ..., \n       [ nan,  nan,  nan, ...,  nan,  nan,  nan],\n       [ nan,  nan,  nan, ...,  nan,  nan,  nan],\n       [ nan,  nan,  nan, ...,  nan,  nan,  nan]])\nCoordinates:\n    longitude    (cast_number) float64 -88.7 -88.7 -88.69 -88.68 -88.67 ...\n    latitude     (cast_number) float64 28.83 28.83 28.83 28.83 28.83 28.82 ...\n  * depths       (depths) float64 0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 ...\n  * cast_number  (cast_number) int64 1926 1927 1928 1929 1930 1931 1932 1933 ...\n    time         (cast_number) float64 36.17 36.17 36.17 36.18 36.18 36.18 ...\nAttributes:\n    units:          C\n    standard_name:  sea_water_temperature\n    long_name:      T [^oC]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "collapsed": false,
        "scrolled": false
      },
      "cell_type": "code",
      "source": "for td in tds:\n    cg = xr.open_dataset('netcdf/Survey%s.nc'%td)\n    print(cg.summary)",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Survey A of Laser 2016 MVP Survey; Large-scale survey before first array of floats was deployed\nSurvey B of Laser 2016 MVP Survey; Cloverleaf pattern while drifters deployed\nSurvey C of Laser 2016 MVP Survey; Run onshore to NW\nSurvey D of Laser 2016 MVP Survey; Survey of messy area on continental slope break\nSurvey E of Laser 2016 MVP Survey; Frontal survey tracking floats in front that runs to south\nSurvey F of Laser 2016 MVP Survey; Frontal survey tracking floats in front that runs to south\nSurvey G of Laser 2016 MVP Survey; Frontal survey tracking floats; stayed in one location and let front advect past\nSurvey H of Laser 2016 MVP Survey; Frontal survey tracking floats; stayed in one location and let front advect past\nSurvey I of Laser 2016 MVP Survey; Vortex to north near continental slope\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "collapsed": false,
        "trusted": true
      },
      "cell_type": "code",
      "source": "dat = xr.open_dataset('netcdf/SurveyA.nc')\nprint(dat)\ndat.close()\ndat = xr.open_dataset('netcdf/SurveyB.nc')\nprint(dat)\ndat.close()",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": "<xarray.Dataset>\nDimensions:      (cast_number: 167, depths: 321)\nCoordinates:\n    longitude    (cast_number) float64 -87.36 -87.37 -87.37 -87.38 -87.39 ...\n    latitude     (cast_number) float64 28.19 28.2 28.21 28.23 28.24 28.25 ...\n  * depths       (depths) float64 0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 ...\n  * cast_number  (cast_number) int64 8 9 10 11 12 13 14 15 16 17 18 19 20 21 ...\n    time         (cast_number) datetime64[ns] 2016-01-20T15:43:00.163200 ...\nData variables:\n    temp         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    cond         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    salinity     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pden         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pressure     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    analog       (depths, cast_number) float64 nan nan nan nan nan nan nan ...\nAttributes:\n    title:                LA16: Survey A\n    summary:              Survey A of Laser 2016 MVP Survey; Large-scale surv...\n    history:              Version to GRIIDC, June/July 2017\n    creator_name:         Jody Klymak\n    creator_email:        jklymak@uvic.ca\n    institution:          U. of Victoria\n    source:               UVic Moving Vessel Profiler\n    processing_level:     medium: first pass and some science has been done w...\n    time_coverage_start:  2016-01-20T15:43:00.163200000\n    time_coverage_end:    2016-01-21T08:41:36.499200000\n    featureType:          Profile\n    comment:              Made with MVP_dataprocess.py https://github.com/jkl...\n    Conventions:          CF-1.6\n    references:           Cruise report: http://web.uvic.ca/~jklymak/LA16\n<xarray.Dataset>\nDimensions:      (cast_number: 63, depths: 321)\nCoordinates:\n    longitude    (cast_number) float64 -87.68 -87.67 -87.66 -87.65 -87.64 ...\n    latitude     (cast_number) float64 29.04 29.04 29.04 29.05 29.05 29.06 ...\n  * depths       (depths) float64 0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 ...\n  * cast_number  (cast_number) int64 178 179 180 181 182 183 184 185 186 187 ...\n    time         (cast_number) datetime64[ns] 2016-01-21T13:29:09.801599999 ...\nData variables:\n    temp         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    cond         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    salinity     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pden         (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    pressure     (depths, cast_number) float64 nan nan nan nan nan nan nan ...\n    analog       (depths, cast_number) float64 nan nan nan nan nan nan nan ...\nAttributes:\n    title:                LA16: Survey B\n    summary:              Survey B of Laser 2016 MVP Survey; Cloverleaf patte...\n    history:              Version to GRIIDC, June/July 2017\n    creator_name:         Jody Klymak\n    creator_email:        jklymak@uvic.ca\n    institution:          U. of Victoria\n    source:               UVic Moving Vessel Profiler\n    processing_level:     medium: first pass and some science has been done w...\n    time_coverage_start:  2016-01-21T13:29:09.801599999\n    time_coverage_end:    2016-01-21T18:27:05.961600000\n    featureType:          Profile\n    comment:              Made with MVP_dataprocess.py https://github.com/jkl...\n    Conventions:          CF-1.6\n    references:           Cruise report: http://web.uvic.ca/~jklymak/LA16\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "collapsed": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.0",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "toc": {
      "threshold": 4,
      "number_sections": true,
      "toc_cell": false,
      "toc_window_display": false,
      "toc_section_display": "block",
      "sideBar": true,
      "navigate_menu": true,
      "moveMenuLeft": true,
      "widenNotebook": false,
      "colors": {
        "hover_highlight": "#DAA520",
        "selected_highlight": "#FFD700",
        "running_highlight": "#FF0000"
      },
      "nav_menu": {
        "height": "290px",
        "width": "252px"
      }
    },
    "gist": {
      "id": "8de1b93d8203e2d5cad855bead046bbe",
      "data": {
        "description": "Make CF-compliant netcdf MVP files",
        "public": true
      }
    },
    "_draft": {
      "nbviewer_url": "https://gist.github.com/8de1b93d8203e2d5cad855bead046bbe"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}