Esmlab scripts: https://mybinder.org/v2/gist/andersy005/ba6957612ad97dc56044d63c9e7d9226/master

ann_mean.ipynb

annual_mean.ipynb

cesm1-le-collection.yml

name: cesm1-le
collection_type: cesm
include_cache_dir: true
data_sources:
  CTRL:
    locations:
      - name: GLADE
        loc_type: posix
        direct_access: True
        urlpath: /glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE
      - name: HPSS
        loc_type: hsi
        direct_access: False
        urlpath: /CCSM/csm/CESM-CAM5-BGC-LE
    component_attrs:
      ocn:
        grid: POP_gx1v6
    case_members:
      - case: b.e11.B1850C5CN.f09_g16.005
        sequence_order: 0
        ensemble: 0
        has_ocean_bgc: True
        year_offset: 1448

  20C:
    locations:
      - name: GLADE
        loc_type: posix
        direct_access: True
        urlpath: /glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE
      - name: HPSS
        loc_type: hsi
        direct_access: False
        urlpath: /CCSM/csm/CESM-CAM5-BGC-LE
    component_attrs:
      ocn:
        grid: POP_gx1v6
    case_members:
      - case: b.e11.B20TRC5CNBDRD.f09_g16.001
        sequence_order: 0
        ensemble: 1
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.002
        sequence_order: 0
        ensemble: 2
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.003
        sequence_order: 0
        ensemble: 3
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.004
        sequence_order: 0
        ensemble: 4
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.005
        sequence_order: 0
        ensemble: 5
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.006
        sequence_order: 0
        ensemble: 6
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.007
        sequence_order: 0
        ensemble: 7
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.008
        sequence_order: 0
        ensemble: 8
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.009
        sequence_order: 0
        ensemble: 9
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.010
        sequence_order: 0
        ensemble: 10
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.011
        sequence_order: 0
        ensemble: 11
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.012
        sequence_order: 0
        ensemble: 12
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.013
        sequence_order: 0
        ensemble: 13
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.014
        sequence_order: 0
        ensemble: 14
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.015
        sequence_order: 0
        ensemble: 15
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.016
        sequence_order: 0
        ensemble: 16
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.017
        sequence_order: 0
        ensemble: 17
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.018
        sequence_order: 0
        ensemble: 18
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.019
        sequence_order: 0
        ensemble: 19
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.020
        sequence_order: 0
        ensemble: 20
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.021
        sequence_order: 0
        ensemble: 21
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.022
        sequence_order: 0
        ensemble: 22
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.023
        sequence_order: 0
        ensemble: 23
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.024
        sequence_order: 0
        ensemble: 24
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.025
        sequence_order: 0
        ensemble: 25
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.026
        sequence_order: 0
        ensemble: 26
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.027
        sequence_order: 0
        ensemble: 27
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.028
        sequence_order: 0
        ensemble: 28
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.029
        sequence_order: 0
        ensemble: 29
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.030
        sequence_order: 0
        ensemble: 30
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.031
        sequence_order: 0
        ensemble: 31
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.032
        sequence_order: 0
        ensemble: 32
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.033
        sequence_order: 0
        ensemble: 33
        has_ocean_bgc: False
      - case: b.e11.B20TRC5CNBDRD.f09_g16.034
        sequence_order: 0
        ensemble: 34
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.035
        sequence_order: 0
        ensemble: 35
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.101
        sequence_order: 0
        ensemble: 101
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.102
        sequence_order: 0
        ensemble: 102
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.103
        sequence_order: 0
        ensemble: 103
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.104
        sequence_order: 0
        ensemble: 104
        has_ocean_bgc: True
      - case: b.e11.B20TRC5CNBDRD.f09_g16.105
        sequence_order: 0
        ensemble: 105
        has_ocean_bgc: True
  RCP85:
    locations:
      - name: GLADE
        loc_type: posix
        direct_access: True
        urlpath: /glade/collections/cdg/data/cesmLE/CESM-CAM5-BGC-LE
      - name: HPSS
        loc_type: hsi
        direct_access: False
        urlpath: /CCSM/csm/CESM-CAM5-BGC-LE
    component_attrs:
      ocn:
        grid: POP_gx1v6
    case_members:
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.001
        sequence_order: 1
        ensemble: 1
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.002
        sequence_order: 1
        ensemble: 2
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.003
        sequence_order: 1
        ensemble: 3
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.004
        sequence_order: 1
        ensemble: 4
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.005
        sequence_order: 1
        ensemble: 5
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.006
        sequence_order: 1
        ensemble: 6
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.007
        sequence_order: 1
        ensemble: 7
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.008
        sequence_order: 1
        ensemble: 8
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.009
        sequence_order: 1
        ensemble: 9
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.010
        sequence_order: 1
        ensemble: 10
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.011
        sequence_order: 1
        ensemble: 11
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.012
        sequence_order: 1
        ensemble: 12
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.013
        sequence_order: 1
        ensemble: 13
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.014
        sequence_order: 1
        ensemble: 14
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.015
        sequence_order: 1
        ensemble: 15
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.016
        sequence_order: 1
        ensemble: 16
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.017
        sequence_order: 1
        ensemble: 17
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.018
        sequence_order: 1
        ensemble: 18
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.019
        sequence_order: 1
        ensemble: 19
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.020
        sequence_order: 1
        ensemble: 20
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.021
        sequence_order: 1
        ensemble: 21
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.022
        sequence_order: 1
        ensemble: 22
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.023
        sequence_order: 1
        ensemble: 23
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.024
        sequence_order: 1
        ensemble: 24
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.025
        sequence_order: 1
        ensemble: 25
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.026
        sequence_order: 1
        ensemble: 26
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.027
        sequence_order: 1
        ensemble: 27
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.028
        sequence_order: 1
        ensemble: 28
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.029
        sequence_order: 1
        ensemble: 29
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.030
        sequence_order: 1
        ensemble: 30
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.031
        sequence_order: 1
        ensemble: 31
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.032
        sequence_order: 1
        ensemble: 32
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.033
        sequence_order: 1
        ensemble: 33
        has_ocean_bgc: False
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.034
        sequence_order: 1
        ensemble: 34
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.035
        sequence_order: 1
        ensemble: 35
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.101
        sequence_order: 1
        ensemble: 101
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.102
        sequence_order: 1
        ensemble: 102
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.103
        sequence_order: 1
        ensemble: 103
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.104
        sequence_order: 1
        ensemble: 104
        has_ocean_bgc: True
      - case: b.e11.BRCP85C5CNBDRD.f09_g16.105
        sequence_order: 1
        ensemble: 105
        has_ocean_bgc: True

compute_ann_mean.ipynb

conftest.py

import sys
import pytest
import cftime
import numpy as np
import numpy.matlib as npm
import xarray as xr


@pytest.fixture
def dset():
    start_date = np.array([0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334], dtype=np.float64)
    start_date = np.append(start_date, start_date + 365)
    end_date = np.array([31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365], dtype=np.float64)
    end_date = np.append(end_date, end_date + 365)
    ds = xr.Dataset(coords={'time': 24, 'lat': 2, 'lon': 2, 'd2': 2})
    ds['time'] = xr.DataArray(end_date, dims='time')
    ds['lat'] = xr.DataArray([0, 1], dims='lat')
    ds['lon'] = xr.DataArray([0, 1], dims='lon')
    ds['d2'] = xr.DataArray([0, 1], dims='d2')
    ds['time_bound'] = xr.DataArray(
        np.array([start_date, end_date]).transpose(), dims=['time', 'd2']
    )
    ds['variable_1'] = xr.DataArray(
        np.append(
            np.zeros([12, 2, 2], dtype='float32'), np.ones([12, 2, 2], dtype='float32'), axis=0
        ),
        dims=['time', 'lat', 'lon'],
    )
    ds['variable_2'] = xr.DataArray(
        np.append(
            np.ones([12, 2, 2], dtype='float32'), np.zeros([12, 2, 2], dtype='float32'), axis=0
        ),
        dims=['time', 'lat', 'lon'],
    )
    ds.time.attrs['units'] = 'days since 0001-01-01 00:00:00'
    ds.time.attrs['calendar'] = 'noleap'
    ds.time.attrs['bounds'] = 'time_bound'
    return ds.copy(True)

def xr_ds_ex(decode_times=True, nyrs=3, var_const=True):
    """return an example xarray.Dataset object, useful for testing functions"""

    # set up values for Dataset, 4 yrs of analytic monthly values
    days_1yr = np.array([31.0, 28.0, 31.0, 30.0, 31.0, 30.0, 31.0, 31.0, 30.0, 31.0, 30.0, 31.0])
    time_edges = np.insert(
        np.cumsum(npm.repmat(days_1yr, nyrs, 1)), 0, 0)
    time_bounds_vals = np.stack((time_edges[:-1], time_edges[1:]), axis=1)
    time_vals = np.mean(time_bounds_vals, axis=1)
    time_vals_yr = time_vals / 365.0
    if var_const:
        var_vals = np.ones_like(time_vals_yr)
    else:
        var_vals = np.sin(np.pi * time_vals_yr) * np.exp(-0.1 * time_vals_yr)

    time_units = 'days since 0001-01-01'
    calendar = 'noleap'

    if decode_times:
        time_vals = cftime.num2date(time_vals, time_units, calendar)
        time_bounds_vals = cftime.num2date(time_bounds_vals, time_units, calendar)

    # create Dataset, including time_bounds
    time_var = xr.DataArray(time_vals, name='time', dims='time', coords={'time':time_vals},
                            attrs={'bounds':'time_bounds'})
    if not decode_times:
        time_var.attrs['units'] = time_units
        time_var.attrs['calendar'] = calendar
    time_bounds = xr.DataArray(time_bounds_vals, name='time_bounds', dims=('time', 'd2'),
                               coords={'time':time_var})
    var = xr.DataArray(var_vals, name='var_ex', dims='time', coords={'time':time_var})
    ds = var.to_dataset()
    ds = xr.merge((ds, time_bounds))

    if decode_times:
        ds.time.encoding['units'] = time_units
        ds.time.encoding['calendar'] = calendar

    return ds

environment.yml

name: esmlab
channels:
  - conda-forge
dependencies:
  - python=3.7
  - xarray
  - cftime
  - numpy
  - dask 
  

methods.txt

['compute_time',
 'compute_time_bound_diff',
 'compute_time_var',
 'decode_arbitrary_time',
 'get_original_metadata',
 'get_time_decoded',
 'get_time_undecoded',
 'get_variables',
 'infer_time_bound_var',
 'infer_time_coord_name',
 'isdecoded',
 'restore_dataset',
 'sel_time',
 'set_time',
 'time_year_to_midyeardate',
 'uncompute_time_var',
 'update_metadata']

stats_tests.ipynb

test_climatology.py

#!/usr/bin/env python
from __future__ import absolute_import, division, print_function

import os

import numpy as np
import pytest
import xarray as xr
import itertools

from esmlab.climatology import compute_ann_mean, compute_mon_anomaly, compute_mon_climatology
from esmlab.datasets import open_dataset

functions = [compute_mon_climatology, compute_ann_mean, compute_mon_anomaly]
decoded = [True, False]
dsets = ['tiny', 'cesm_pop_daily']
params = list(itertools.product(dsets, decoded, functions))

@pytest.mark.parametrize('ds, decoded, function', params)
def test_compute_climatology(ds, decoded, function):
    dset = open_dataset(ds, decode_times=decoded)
    computed_dset = function(dset)
    assert isinstance(computed_dset, xr.Dataset)
    assert computed_dset.time.dtype == dset.time.dtype
    for key, value in dset.time.attrs.items():
        assert key in computed_dset.time.attrs
        assert value == computed_dset.time.attrs[key]


@pytest.mark.parametrize('ds, decoded, function', params)
def test_compute_climatology_drop_time_bound(ds, decoded, function):
    dset = open_dataset(ds, decode_times=decoded)
    dset_time_bound = dset.time.attrs['bounds']
    dset = dset.drop(dset_time_bound)
    del dset.time.attrs['bounds']
    
    computed_dset = function(dset)
    assert isinstance(computed_dset, xr.Dataset)
    assert computed_dset.time.dtype == dset.time.dtype
    if function == compute_mon_anomaly:
        assert (dset.time.values == computed_dset.time.values).all()
    for key, value in dset.time.attrs.items():
        assert key in computed_dset.time.attrs
        assert value == computed_dset.time.attrs[key]


@pytest.mark.parametrize('ds', ['tiny', 'cesm_cice_daily'])
def test_compute_climatology_daisy_chain(ds):
    dset = open_dataset(ds, decode_times=False)

    computed_dset = compute_mon_climatology(dset)
    assert isinstance(computed_dset, xr.Dataset)

    computed_dset2 = compute_mon_anomaly(computed_dset)
    assert isinstance(computed_dset2, xr.Dataset)

    computed_dset3 = compute_ann_mean(computed_dset)
    assert isinstance(computed_dset3, xr.Dataset)

    computed_dset3 = compute_ann_mean(computed_dset2)
    assert isinstance(computed_dset3, xr.Dataset)


def test_compute_mon_climatology_values(dset):
    computed_dset = compute_mon_climatology(dset)
    np.testing.assert_equal(computed_dset.variable_1.values, 0.5)
    assert computed_dset.time.dtype == dset.time.dtype


def test_compute_mon_anomaly_values(dset):
    computed_dset = compute_mon_anomaly(dset)
    assert isinstance(computed_dset, xr.Dataset)
    a = [-0.5] * 48
    b = [0.5] * 48
    a.extend(b)
    expected = np.array(a)
    np.testing.assert_equal(computed_dset.variable_1.values.ravel(), expected)
    assert computed_dset.time.dtype == dset.time.dtype
    assert (dset.time.values == computed_dset.time.values).all()
    for key, value in dset.time.attrs.items():
        assert key in computed_dset.time.attrs
        assert value == computed_dset.time.attrs[key]


def test_compute_ann_mean_values(dset):
    computed_dset = compute_ann_mean(dset)
    assert isinstance(computed_dset, xr.Dataset)
    assert computed_dset.time.dtype == dset.time.dtype
    expected = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0])
    np.testing.assert_equal(computed_dset.variable_1.values.ravel(), expected)


def test_compute_ann_mean_values_missing(dset):
    dset.variable_1.values[0:3, :, :] = np.nan
    computed_dset = compute_ann_mean(dset)
    assert isinstance(computed_dset, xr.Dataset)
    assert computed_dset.time.dtype == dset.time.dtype
    expected = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0])
    np.testing.assert_equal(computed_dset.variable_1.values.ravel(), expected)