Skip to content

Instantly share code, notes, and snippets.

@andersy005

andersy005/esmlab-data-generation.ipynb

Created March 3, 2019 22:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save andersy005/500d58f19d6f678d07e3ed57d61b493b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save andersy005/500d58f19d6f678d07e3ed57d61b493b to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
esmlab-data-generation.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import xarray as xr\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## CESM-CICE"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"root_dir = \"/gpfs/fs1/p/cesm/pcwg/timeseries-cmip6/b.e21.B1850.f09_g17.CMIP6-piControl.001/ice/proc/tseries\""
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
"case = \"b.e21.B1850.f09_g17.CMIP6-piControl.001\"\n",
"year1 = \"0061\"\n",
"year2 = \"0099\"\n",
"year1m = \"0001\"\n",
"year2m = \"0099\""
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [],
"source": [
"filename_month = f\"{root_dir}/month_1/\"+case+\".cice.h.aicen.\"+year1m+\"01-\"+year2m+\"12.nc\"\n",
"filename_day = f\"{root_dir}/day_1/\"+case+\".cice.h1.aicen_d.\"+year1+\"0101-\"+year2+\"1231.nc\"\n",
"filename1 = f\"{root_dir}/day_1/\"+case+\".cice.h1.aicen_d_fix.\"+year1+\"0101-\"+year2+\"1231.nc\"\n",
"filename2 = f\"{root_dir}/day_1/\"+case+\".cice.h.aicen_fix.\"+year1m+\"01-\"+year2m+\"12.nc\"\n"
]
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {},
"outputs": [],
"source": [
"def open_dataset(file_path, decode_times=False, decode_coords=False):\n",
" return xr.open_dataset(file_path, decode_times=decode_times, decode_coords=decode_coords)"
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {},
"outputs": [],
"source": [
"ds = open_dataset(filename1)\n",
"ds_day = open_dataset(filename_day)\n",
"ds_month = open_dataset(filename_month)"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (d2: 2, nc: 5, ni: 320, nj: 384, nkbio: 5, nkice: 8, nksnow: 3, nvertices: 4, time: 14235)\n",
"Coordinates:\n",
" * time (time) float32 21901.0 21902.0 21903.0 ... 36134.0 36135.0\n",
"Dimensions without coordinates: d2, nc, ni, nj, nkbio, nkice, nksnow, nvertices\n",
"Data variables:\n",
" TLON (nj, ni) float32 ...\n",
" TLAT (nj, ni) float32 ...\n",
" ULON (nj, ni) float32 ...\n",
" ULAT (nj, ni) float32 ...\n",
" NCAT (nc) float32 ...\n",
" VGRDi (nkice) float32 ...\n",
" VGRDs (nksnow) float32 ...\n",
" VGRDb (nkbio) float32 ...\n",
" tmask (nj, ni) float32 ...\n",
" tarea (nj, ni) float32 ...\n",
" uarea (nj, ni) float32 ...\n",
" dxt (nj, ni) float32 ...\n",
" dyt (nj, ni) float32 ...\n",
" dxu (nj, ni) float32 ...\n",
" dyu (nj, ni) float32 ...\n",
" HTN (nj, ni) float32 ...\n",
" HTE (nj, ni) float32 ...\n",
" ANGLE (nj, ni) float32 ...\n",
" ANGLET (nj, ni) float32 ...\n",
" lont_bounds (nj, ni, nvertices) float32 ...\n",
" latt_bounds (nj, ni, nvertices) float32 ...\n",
" lonu_bounds (nj, ni, nvertices) float32 ...\n",
" latu_bounds (nj, ni, nvertices) float32 ...\n",
" time_bounds (time, d2) float32 ...\n",
" aicen_d (time, nc, nj, ni) float32 ...\n",
"Attributes:\n",
" title: b.e21.B1850.f09_g17.CMIP6-piControl.001\n",
" contents: Diagnostic and Prognostic Variables\n",
" source: Los Alamos Sea Ice Model (CICE) Version 5\n",
" time_period_freq: day_1\n",
" model_doi_url: https://doi.org/10.5065/D67H1H0V\n",
" comment: All years have exactly 365 days\n",
" comment2: File written on model date 00610102\n",
" comment3: seconds elapsed into model date: 0\n",
" conventions: CF-1.0\n",
" history: This dataset was created on 2018-08-12 at 13:23\n",
" io_flavor: io_pio"
]
},
"execution_count": 76,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds_day"
]
},
{
"cell_type": "code",
"execution_count": 77,
"metadata": {},
"outputs": [],
"source": [
"ds_month.attrs[\"Extracted_from\"] = filename_month\n",
"ds_month.isel(time=slice(0, 24), nc=slice(0, 6), ni=slice(0, 6), nj=slice(0, 6)).to_netcdf(\"/glade/u/home/abanihi/devel/esm/esmlab-data/cesm_cice_monthly.nc\")"
]
},
{
"cell_type": "code",
"execution_count": 78,
"metadata": {},
"outputs": [],
"source": [
"ds_day.attrs[\"Extracted_from\"] = filename_day\n",
"ds_day.isel(time=slice(0, 365), nc=slice(0, 6), ni=slice(0, 6), nj=slice(0, 6)).to_netcdf(\"/glade/u/home/abanihi/devel/esm/esmlab-data/cesm_cice_daily.nc\")"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"OrderedDict([('title', 'b.e21.B1850.f09_g17.CMIP6-piControl.001'),\n",
" ('contents', 'Diagnostic and Prognostic Variables'),\n",
" ('source', 'Los Alamos Sea Ice Model (CICE) Version 5'),\n",
" ('time_period_freq', 'month_1'),\n",
" ('model_doi_url', 'https://doi.org/10.5065/D67H1H0V'),\n",
" ('comment', 'All years have exactly 365 days'),\n",
" ('comment2', 'File written on model date 00010201'),\n",
" ('comment3', 'seconds elapsed into model date: 0'),\n",
" ('conventions', 'CF-1.0'),\n",
" ('history', 'This dataset was created on 2018-08-09 at 18:18'),\n",
" ('io_flavor', 'io_pio')])"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## CESM-POP"
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
"root_dir= \"/glade/collections/cdg/timeseries-cmip6/b.e21.B1850.f09_g17.CMIP6-piControl.001/ocn/proc/tseries\""
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['month_1', 'day_1', 'year_1']"
]
},
"execution_count": 60,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.listdir(root_dir)"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {},
"outputs": [],
"source": [
"filename_month = f\"{root_dir}/month_1/b.e21.B1850.f09_g17.CMIP6-piControl.001.pop.h.NO3.050001-059912.nc\"\n",
"filename_day = f\"{root_dir}/day_1/b.e21.B1850.f09_g17.CMIP6-piControl.001.pop.h.ecosys.nday1.FG_CO2_2.05000101-05991231.nc\"\n",
"filename_year = f\"{root_dir}/year_1/b.e21.B1850.f09_g17.CMIP6-piControl.001.pop.h.ecosys.nyear1.CaCO3_FLUX_IN.0800-0899.nc\""
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (d2: 2, lat_aux_grid: 395, moc_comp: 3, moc_z: 61, nlat: 384, nlon: 320, time: 1200, transport_comp: 5, transport_reg: 2, z_t: 60, z_t_150m: 15, z_w: 60, z_w_bot: 60, z_w_top: 60)\n",
"Coordinates:\n",
" * z_t (z_t) float32 500.0 1500.0 ... 512502.8 537500.0\n",
" * z_t_150m (z_t_150m) float32 500.0 1500.0 ... 13500.0 14500.0\n",
" * z_w (z_w) float32 0.0 1000.0 ... 500004.7 525000.94\n",
" * z_w_top (z_w_top) float32 0.0 1000.0 ... 500004.7 525000.94\n",
" * z_w_bot (z_w_bot) float32 1000.0 2000.0 ... 549999.06\n",
" * lat_aux_grid (lat_aux_grid) float32 -79.48815 -78.952896 ... 90.0\n",
" * moc_z (moc_z) float32 0.0 1000.0 ... 525000.94 549999.06\n",
" ULONG (nlat, nlon) float64 ...\n",
" ULAT (nlat, nlon) float64 ...\n",
" TLONG (nlat, nlon) float64 ...\n",
" TLAT (nlat, nlon) float64 ...\n",
" * time (time) object 0500-02-01 00:00:00 ... 0600-01-01 00:00:00\n",
"Dimensions without coordinates: d2, moc_comp, nlat, nlon, transport_comp, transport_reg\n",
"Data variables:\n",
" moc_components (moc_comp) |S384 ...\n",
" transport_components (transport_comp) |S384 ...\n",
" transport_regions (transport_reg) |S384 ...\n",
" dz (z_t) float32 ...\n",
" dzw (z_w) float32 ...\n",
" KMT (nlat, nlon) float64 ...\n",
" KMU (nlat, nlon) float64 ...\n",
" REGION_MASK (nlat, nlon) float64 ...\n",
" UAREA (nlat, nlon) float64 ...\n",
" TAREA (nlat, nlon) float64 ...\n",
" HU (nlat, nlon) float64 ...\n",
" HT (nlat, nlon) float64 ...\n",
" DXU (nlat, nlon) float64 ...\n",
" DYU (nlat, nlon) float64 ...\n",
" DXT (nlat, nlon) float64 ...\n",
" DYT (nlat, nlon) float64 ...\n",
" HTN (nlat, nlon) float64 ...\n",
" HTE (nlat, nlon) float64 ...\n",
" HUS (nlat, nlon) float64 ...\n",
" HUW (nlat, nlon) float64 ...\n",
" ANGLE (nlat, nlon) float64 ...\n",
" ANGLET (nlat, nlon) float64 ...\n",
" days_in_norm_year timedelta64[ns] ...\n",
" grav float64 ...\n",
" omega float64 ...\n",
" radius float64 ...\n",
" cp_sw float64 ...\n",
" sound float64 ...\n",
" vonkar float64 ...\n",
" cp_air float64 ...\n",
" rho_air float64 ...\n",
" rho_sw float64 ...\n",
" rho_fw float64 ...\n",
" stefan_boltzmann float64 ...\n",
" latent_heat_vapor float64 ...\n",
" latent_heat_fusion float64 ...\n",
" latent_heat_fusion_mks float64 ...\n",
" ocn_ref_salinity float64 ...\n",
" sea_ice_salinity float64 ...\n",
" T0_Kelvin float64 ...\n",
" salt_to_ppt float64 ...\n",
" ppt_to_salt float64 ...\n",
" mass_to_Sv float64 ...\n",
" heat_to_PW float64 ...\n",
" salt_to_Svppt float64 ...\n",
" salt_to_mmday float64 ...\n",
" momentum_factor float64 ...\n",
" hflux_factor float64 ...\n",
" fwflux_factor float64 ...\n",
" salinity_factor float64 ...\n",
" sflux_factor float64 ...\n",
" nsurface_t float64 ...\n",
" nsurface_u float64 ...\n",
" time_bound (time, d2) object ...\n",
" NO3 (time, z_t, nlat, nlon) float32 ...\n",
"Attributes:\n",
" title: b.e21.B1850.f09_g17.CMIP6-piControl.001\n",
" history: none\n",
" Conventions: CF-1.0; http://www.cgd.ucar.edu/cms/eaton/netcdf/CF-cu...\n",
" time_period_freq: month_1\n",
" model_doi_url: https://doi.org/10.5065/D67H1H0V\n",
" contents: Diagnostic and Prognostic Variables\n",
" source: CCSM POP2, the CCSM Ocean Component\n",
" revision: $Id: tavg.F90 89644 2018-08-04 14:26:01Z klindsay $\n",
" calendar: All years have exactly 365 days.\n",
" start_time: This dataset was created on 2018-09-24 at 08:32:53.4\n",
" cell_methods: cell_methods = time: mean ==> the variable values are ..."
]
},
"execution_count": 62,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"xr.open_dataset(filename_month)"
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [],
"source": [
"ds_year = open_dataset(filename_year)\n",
"ds_day = open_dataset(filename_day)\n",
"ds_month = open_dataset(filename_month)"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (d2: 2, nlat: 384, nlon: 320, time: 100, z_t: 60, z_t_150m: 15, z_w: 60, z_w_bot: 60, z_w_top: 60)\n",
"Coordinates:\n",
" * z_t (z_t) float32 500.0 1500.0 ... 512502.8 537500.0\n",
" * z_t_150m (z_t_150m) float32 500.0 1500.0 ... 13500.0 14500.0\n",
" * z_w (z_w) float32 0.0 1000.0 ... 500004.7 525000.94\n",
" * z_w_top (z_w_top) float32 0.0 1000.0 ... 500004.7 525000.94\n",
" * z_w_bot (z_w_bot) float32 1000.0 2000.0 ... 549999.06\n",
" * time (time) float64 2.924e+05 2.927e+05 ... 3.285e+05\n",
"Dimensions without coordinates: d2, nlat, nlon\n",
"Data variables:\n",
" dz (z_t) float32 ...\n",
" dzw (z_w) float32 ...\n",
" ULONG (nlat, nlon) float64 ...\n",
" ULAT (nlat, nlon) float64 ...\n",
" TLONG (nlat, nlon) float64 ...\n",
" TLAT (nlat, nlon) float64 ...\n",
" KMT (nlat, nlon) float64 ...\n",
" KMU (nlat, nlon) float64 ...\n",
" REGION_MASK (nlat, nlon) float64 ...\n",
" UAREA (nlat, nlon) float64 ...\n",
" TAREA (nlat, nlon) float64 ...\n",
" HU (nlat, nlon) float64 ...\n",
" HT (nlat, nlon) float64 ...\n",
" DXU (nlat, nlon) float64 ...\n",
" DYU (nlat, nlon) float64 ...\n",
" DXT (nlat, nlon) float64 ...\n",
" DYT (nlat, nlon) float64 ...\n",
" HTN (nlat, nlon) float64 ...\n",
" HTE (nlat, nlon) float64 ...\n",
" HUS (nlat, nlon) float64 ...\n",
" HUW (nlat, nlon) float64 ...\n",
" ANGLE (nlat, nlon) float64 ...\n",
" ANGLET (nlat, nlon) float64 ...\n",
" days_in_norm_year float64 ...\n",
" grav float64 ...\n",
" omega float64 ...\n",
" radius float64 ...\n",
" cp_sw float64 ...\n",
" sound float64 ...\n",
" vonkar float64 ...\n",
" cp_air float64 ...\n",
" rho_air float64 ...\n",
" rho_sw float64 ...\n",
" rho_fw float64 ...\n",
" stefan_boltzmann float64 ...\n",
" latent_heat_vapor float64 ...\n",
" latent_heat_fusion float64 ...\n",
" latent_heat_fusion_mks float64 ...\n",
" ocn_ref_salinity float64 ...\n",
" sea_ice_salinity float64 ...\n",
" T0_Kelvin float64 ...\n",
" salt_to_ppt float64 ...\n",
" ppt_to_salt float64 ...\n",
" mass_to_Sv float64 ...\n",
" heat_to_PW float64 ...\n",
" salt_to_Svppt float64 ...\n",
" salt_to_mmday float64 ...\n",
" momentum_factor float64 ...\n",
" hflux_factor float64 ...\n",
" fwflux_factor float64 ...\n",
" salinity_factor float64 ...\n",
" sflux_factor float64 ...\n",
" nsurface_t float64 ...\n",
" nsurface_u float64 ...\n",
" time_bound (time, d2) float64 ...\n",
" CaCO3_FLUX_IN (time, z_t, nlat, nlon) float32 ...\n",
"Attributes:\n",
" title: b.e21.B1850.f09_g17.CMIP6-piControl.001\n",
" history: none\n",
" Conventions: CF-1.0; http://www.cgd.ucar.edu/cms/eaton/netcdf/CF-cu...\n",
" time_period_freq: year_1\n",
" model_doi_url: https://doi.org/10.5065/D67H1H0V\n",
" contents: Diagnostic and Prognostic Variables\n",
" source: CCSM POP2, the CCSM Ocean Component\n",
" revision: $Id: tavg.F90 89644 2018-08-04 14:26:01Z klindsay $\n",
" calendar: All years have exactly 365 days.\n",
" start_time: This dataset was created on 2018-10-11 at 13:36:05.6\n",
" cell_methods: cell_methods = time: mean ==> the variable values are ..."
]
},
"execution_count": 65,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds_year"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"ds_month.attrs[\"Extracted_from\"] = filename_month\n",
"ds_day.attrs[\"Extracted_from\"] = filename_day\n",
"ds_year.attrs[\"Extracted_from\"] = filename_year\n",
"ds_year.isel(time=slice(0, 10), z_t=slice(0, 6), nlat=slice(0, 6), nlon=slice(0, 6)).to_netcdf(\"/glade/u/home/abanihi/devel/esm/esmlab-data/cesm_pop_yearly.nc\")\n",
"ds_day.isel(time=slice(0, 365), z_t=slice(0, 6), nlat=slice(0, 6), nlon=slice(0, 6)).to_netcdf(\"/glade/u/home/abanihi/devel/esm/esmlab-data/cesm_pop_daily.nc\")\n",
"ds_month.isel(time=slice(0, 24), z_t=slice(0, 6), nlat=slice(0, 6), nlon=slice(0, 6)).to_netcdf(\"/glade/u/home/abanihi/devel/esm/esmlab-data/cesm_pop_monthly.nc\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:pangeo]",
"language": "python",
"name": "conda-env-pangeo-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment