Skip to content

Instantly share code, notes, and snippets.

@omad

omad/mogreps ingest.ipynb

Last active February 20, 2018 23:48
Show Gist options
  • Save omad/60e755849d1608fea6765aabff65ff88 to your computer and use it in GitHub Desktop.
Save omad/60e755849d1608fea6765aabff65ff88 to your computer and use it in GitHub Desktop.
Download ZIP
Script for creating product and dataset documents for a mogreps NetCDF file
Display the source blob
Display the rendered blob
Raw
mogreps ingest.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"import netCDF4\n",
"from itertools import groupby\n",
"from operator import attrgetter\n",
"from pathlib import Path\n",
"from dateutil.parser import parse\n",
"from datetime import datetime\n",
"import uuid"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"filename = '/Users/omad/20180205T0000Z_tropical_africa.nc'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"nco = netCDF4.Dataset(filename)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"odict_keys(['air_pressure_at_sea_level', 'latitude_longitude', 'time', 'latitude', 'longitude', 'forecast_period', 'forecast_reference_time', 'air_temperature', 'time_0', 'forecast_period_0', 'height', 'moisture_content_of_soil_layer', 'soil_model_level_number', 'relative_humidity', 'snowfall_amount', 'time_1', 'time_1_bnds', 'forecast_period_1', 'forecast_period_1_bnds', 'stratiform_rainfall_amount', 'forecast_period_2', 'forecast_period_2_bnds', 'time_2', 'time_2_bnds', 'stratiform_rainfall_amount_0', 'stratiform_rainfall_rate', 'time_3', 'forecast_period_3', 'stratiform_rainfall_rate_0', 'surface_downwelling_longwave_flux', 'time_4', 'forecast_period_4', 'surface_downwelling_shortwave_flux_in_air', 'surface_temperature', 'surface_upward_latent_heat_flux', 'surface_upward_sensible_heat_flux', 'toa_outgoing_longwave_flux', 'time_5', 'forecast_period_5', 'toa_outgoing_shortwave_flux', 'x_wind', 'latitude_0', 'longitude_0', 'height_0', 'y_wind'])"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nco.variables.keys()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'air_pressure_at_sea_level': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 air_pressure_at_sea_level(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_pressure_at_sea_level\n",
" units: Pa\n",
" um_stash_source: m01s16i222\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
" unlimited dimensions: time\n",
" current shape = (41, 763, 763)\n",
" filling on, 'air_temperature': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 air_temperature(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_temperature\n",
" units: K\n",
" um_stash_source: m01s03i236\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 763, 763)\n",
" filling on, 'moisture_content_of_soil_layer': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 moisture_content_of_soil_layer(time_0, soil_model_level_number, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: moisture_content_of_soil_layer\n",
" units: kg m-2\n",
" um_stash_source: m01s08i223\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 4, 763, 763)\n",
" filling on, 'relative_humidity': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 relative_humidity(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: relative_humidity\n",
" units: %\n",
" um_stash_source: m01s03i245\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 763, 763)\n",
" filling on, 'snowfall_amount': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 snowfall_amount(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: snowfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s00i023\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
" unlimited dimensions: time_1\n",
" current shape = (40, 763, 763)\n",
" filling on, 'stratiform_rainfall_amount': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_amount(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
" unlimited dimensions: dim0\n",
" current shape = (40, 763, 763)\n",
" filling on, 'stratiform_rainfall_amount_0': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_amount_0(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
" unlimited dimensions: dim0\n",
" current shape = (40, 763, 763)\n",
" filling on, 'stratiform_rainfall_rate': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_rate(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on, 'stratiform_rainfall_rate_0': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_rate_0(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
" unlimited dimensions: time_1\n",
" current shape = (40, 763, 763)\n",
" filling on, 'surface_downwelling_longwave_flux': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_downwelling_longwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i207\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on, 'surface_downwelling_shortwave_flux_in_air': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_downwelling_shortwave_flux_in_air(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_shortwave_flux_in_air\n",
" units: W m-2\n",
" um_stash_source: m01s01i235\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on, 'surface_temperature': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_temperature(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_temperature\n",
" units: K\n",
" um_stash_source: m01s00i024\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
" unlimited dimensions: time\n",
" current shape = (41, 763, 763)\n",
" filling on, 'surface_upward_latent_heat_flux': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_upward_latent_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_latent_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i234\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on, 'surface_upward_sensible_heat_flux': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_upward_sensible_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_sensible_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i217\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on, 'toa_outgoing_longwave_flux': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 toa_outgoing_longwave_flux(time_5, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i205\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_5 forecast_reference_time\n",
" unlimited dimensions: time_5\n",
" current shape = (40, 763, 763)\n",
" filling on, 'toa_outgoing_shortwave_flux': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 toa_outgoing_shortwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_shortwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s01i208\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on, 'x_wind': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 x_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: x_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i225\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 764, 763)\n",
" filling on, 'y_wind': <class 'netCDF4._netCDF4.Variable'>\n",
" float32 y_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: y_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i226\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 764, 763)\n",
" filling on}"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"interesting_variables = {vname: v for vname, v in nco.variables.items() if 'grid_mapping' in v.ncattrs()}\n",
"interesting_variables"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"air_pressure_at_sea_level\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 air_pressure_at_sea_level(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_pressure_at_sea_level\n",
" units: Pa\n",
" um_stash_source: m01s16i222\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
"unlimited dimensions: time\n",
"current shape = (41, 763, 763)\n",
"filling on\n",
"air_temperature\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 air_temperature(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_temperature\n",
" units: K\n",
" um_stash_source: m01s03i236\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
"unlimited dimensions: time_0\n",
"current shape = (41, 763, 763)\n",
"filling on\n",
"moisture_content_of_soil_layer\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 moisture_content_of_soil_layer(time_0, soil_model_level_number, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: moisture_content_of_soil_layer\n",
" units: kg m-2\n",
" um_stash_source: m01s08i223\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time\n",
"unlimited dimensions: time_0\n",
"current shape = (41, 4, 763, 763)\n",
"filling on\n",
"relative_humidity\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 relative_humidity(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: relative_humidity\n",
" units: %\n",
" um_stash_source: m01s03i245\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
"unlimited dimensions: time_0\n",
"current shape = (41, 763, 763)\n",
"filling on\n",
"snowfall_amount\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 snowfall_amount(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: snowfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s00i023\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
"unlimited dimensions: time_1\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"stratiform_rainfall_amount\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 stratiform_rainfall_amount(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
"unlimited dimensions: dim0\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"stratiform_rainfall_amount_0\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 stratiform_rainfall_amount_0(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
"unlimited dimensions: dim0\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"stratiform_rainfall_rate\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 stratiform_rainfall_rate(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
"unlimited dimensions: time_3\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"stratiform_rainfall_rate_0\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 stratiform_rainfall_rate_0(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
"unlimited dimensions: time_1\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"surface_downwelling_longwave_flux\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 surface_downwelling_longwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i207\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
"unlimited dimensions: time_4\n",
"current shape = (39, 763, 763)\n",
"filling on\n",
"surface_downwelling_shortwave_flux_in_air\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 surface_downwelling_shortwave_flux_in_air(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_shortwave_flux_in_air\n",
" units: W m-2\n",
" um_stash_source: m01s01i235\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
"unlimited dimensions: time_4\n",
"current shape = (39, 763, 763)\n",
"filling on\n",
"surface_temperature\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 surface_temperature(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_temperature\n",
" units: K\n",
" um_stash_source: m01s00i024\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
"unlimited dimensions: time\n",
"current shape = (41, 763, 763)\n",
"filling on\n",
"surface_upward_latent_heat_flux\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 surface_upward_latent_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_latent_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i234\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
"unlimited dimensions: time_3\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"surface_upward_sensible_heat_flux\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 surface_upward_sensible_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_sensible_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i217\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
"unlimited dimensions: time_3\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"toa_outgoing_longwave_flux\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 toa_outgoing_longwave_flux(time_5, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i205\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_5 forecast_reference_time\n",
"unlimited dimensions: time_5\n",
"current shape = (40, 763, 763)\n",
"filling on\n",
"toa_outgoing_shortwave_flux\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 toa_outgoing_shortwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_shortwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s01i208\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
"unlimited dimensions: time_4\n",
"current shape = (39, 763, 763)\n",
"filling on\n",
"x_wind\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 x_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: x_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i225\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
"unlimited dimensions: time_0\n",
"current shape = (41, 764, 763)\n",
"filling on\n",
"y_wind\n",
"<class 'netCDF4._netCDF4.Variable'>\n",
"float32 y_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: y_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i226\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
"unlimited dimensions: time_0\n",
"current shape = (41, 764, 763)\n",
"filling on\n"
]
}
],
"source": [
"for name, var in interesting_variables.items():\n",
" print(name)\n",
" print(var)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('time_4', 'latitude', 'longitude')"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"interesting_variables['toa_outgoing_shortwave_flux'].dimensions"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<itertools.groupby at 0x10dbee278>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groupby(interesting_variables.values(), attrgetter('dimensions'))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('dim0', 'latitude', 'longitude'), ('time', 'latitude', 'longitude'), ('time_0', 'latitude', 'longitude'), ('time_0', 'latitude_0', 'longitude_0'), ('time_0', 'soil_model_level_number', 'latitude', 'longitude'), ('time_1', 'latitude', 'longitude'), ('time_3', 'latitude', 'longitude'), ('time_4', 'latitude', 'longitude'), ('time_5', 'latitude', 'longitude')]\n"
]
}
],
"source": [
"groups = []\n",
"uniquekeys = []\n",
"data = interesting_variables.values()\n",
"keyfunc = attrgetter('dimensions')\n",
"data = sorted(data, key=keyfunc)\n",
"for k, g in groupby(data, keyfunc):\n",
" groups.append(list(g)) # Store group iterator as a list\n",
" uniquekeys.append(k)\n",
"\n",
"print(uniquekeys)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"[2, 2, 2, 2, 1, 2, 3, 3, 1]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[len(g) for g in groups]"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[[<class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_amount(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
" unlimited dimensions: dim0\n",
" current shape = (40, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_amount_0(dim0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s04i201\n",
" cell_methods: time: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_2 forecast_reference_time time_2\n",
" unlimited dimensions: dim0\n",
" current shape = (40, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 air_pressure_at_sea_level(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_pressure_at_sea_level\n",
" units: Pa\n",
" um_stash_source: m01s16i222\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
" unlimited dimensions: time\n",
" current shape = (41, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_temperature(time, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_temperature\n",
" units: K\n",
" um_stash_source: m01s00i024\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period forecast_reference_time\n",
" unlimited dimensions: time\n",
" current shape = (41, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 air_temperature(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: air_temperature\n",
" units: K\n",
" um_stash_source: m01s03i236\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 relative_humidity(time_0, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: relative_humidity\n",
" units: %\n",
" um_stash_source: m01s03i245\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 x_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: x_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i225\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 764, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 y_wind(time_0, latitude_0, longitude_0)\n",
" _FillValue: -1073741800.0\n",
" standard_name: y_wind\n",
" units: m s-1\n",
" um_stash_source: m01s03i226\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time height_0\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 764, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 moisture_content_of_soil_layer(time_0, soil_model_level_number, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: moisture_content_of_soil_layer\n",
" units: kg m-2\n",
" um_stash_source: m01s08i223\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_0 forecast_reference_time\n",
" unlimited dimensions: time_0\n",
" current shape = (41, 4, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 snowfall_amount(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: snowfall_amount\n",
" units: kg m-2\n",
" um_stash_source: m01s00i023\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
" unlimited dimensions: time_1\n",
" current shape = (40, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_rate_0(time_1, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" cell_methods: time_1: mean (interval: 1 hour)\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_1 forecast_reference_time\n",
" unlimited dimensions: time_1\n",
" current shape = (40, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 stratiform_rainfall_rate(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: stratiform_rainfall_rate\n",
" units: kg m-2 s-1\n",
" um_stash_source: m01s04i203\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_upward_latent_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_latent_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i234\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_upward_sensible_heat_flux(time_3, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_upward_sensible_heat_flux\n",
" units: W m-2\n",
" um_stash_source: m01s03i217\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_3 forecast_reference_time\n",
" unlimited dimensions: time_3\n",
" current shape = (40, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_downwelling_longwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i207\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 surface_downwelling_shortwave_flux_in_air(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: surface_downwelling_shortwave_flux_in_air\n",
" units: W m-2\n",
" um_stash_source: m01s01i235\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on, <class 'netCDF4._netCDF4.Variable'>\n",
" float32 toa_outgoing_shortwave_flux(time_4, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_shortwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s01i208\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_4 forecast_reference_time\n",
" unlimited dimensions: time_4\n",
" current shape = (39, 763, 763)\n",
" filling on], [<class 'netCDF4._netCDF4.Variable'>\n",
" float32 toa_outgoing_longwave_flux(time_5, latitude, longitude)\n",
" _FillValue: -1073741800.0\n",
" standard_name: toa_outgoing_longwave_flux\n",
" units: W m-2\n",
" um_stash_source: m01s02i205\n",
" grid_mapping: latitude_longitude\n",
" coordinates: forecast_period_5 forecast_reference_time\n",
" unlimited dimensions: time_5\n",
" current shape = (40, 763, 763)\n",
" filling on]]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groups"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"group_names = [[var.name for var in group] for group in groups]"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"[(('dim0', 'latitude', 'longitude'),\n",
" ['stratiform_rainfall_amount', 'stratiform_rainfall_amount_0']),\n",
" (('time', 'latitude', 'longitude'),\n",
" ['air_pressure_at_sea_level', 'surface_temperature']),\n",
" (('time_0', 'latitude', 'longitude'),\n",
" ['air_temperature', 'relative_humidity']),\n",
" (('time_0', 'latitude_0', 'longitude_0'), ['x_wind', 'y_wind']),\n",
" (('time_0', 'soil_model_level_number', 'latitude', 'longitude'),\n",
" ['moisture_content_of_soil_layer']),\n",
" (('time_1', 'latitude', 'longitude'),\n",
" ['snowfall_amount', 'stratiform_rainfall_rate_0']),\n",
" (('time_3', 'latitude', 'longitude'),\n",
" ['stratiform_rainfall_rate',\n",
" 'surface_upward_latent_heat_flux',\n",
" 'surface_upward_sensible_heat_flux']),\n",
" (('time_4', 'latitude', 'longitude'),\n",
" ['surface_downwelling_longwave_flux',\n",
" 'surface_downwelling_shortwave_flux_in_air',\n",
" 'toa_outgoing_shortwave_flux']),\n",
" (('time_5', 'latitude', 'longitude'), ['toa_outgoing_longwave_flux'])]"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(zip(uniquekeys, group_names))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<class 'netCDF4._netCDF4.Dimension'> (unlimited): name = 'dim0', size = 40"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nco.dimensions['dim0']"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'name': 'stratiform_rainfall_amount_stratiform_rainfall_amount_0', 'description': 'stratiform_rainfall_amount_stratiform_rainfall_amount_0', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'stratiform_rainfall_amount_stratiform_rainfall_amount_0', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'stratiform_rainfall_amount', 'dtype': 'float32', 'units': 'kg m-2', 'nodata': -1073741800.0}, {'name': 'stratiform_rainfall_amount_0', 'dtype': 'float32', 'units': 'kg m-2', 'nodata': -1073741800.0}]}, {'name': 'air_pressure_at_sea_level_surface_temperature', 'description': 'air_pressure_at_sea_level_surface_temperature', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'air_pressure_at_sea_level_surface_temperature', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'air_pressure_at_sea_level', 'dtype': 'float32', 'units': 'Pa', 'nodata': -1073741800.0}, {'name': 'surface_temperature', 'dtype': 'float32', 'units': 'K', 'nodata': -1073741800.0}]}, {'name': 'air_temperature_relative_humidity', 'description': 'air_temperature_relative_humidity', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'air_temperature_relative_humidity', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'air_temperature', 'dtype': 'float32', 'units': 'K', 'nodata': -1073741800.0}, {'name': 'relative_humidity', 'dtype': 'float32', 'units': '%', 'nodata': -1073741800.0}]}, {'name': 'x_wind_y_wind', 'description': 'x_wind_y_wind', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'x_wind_y_wind', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'x_wind', 'dtype': 'float32', 'units': 'm s-1', 'nodata': -1073741800.0}, {'name': 'y_wind', 'dtype': 'float32', 'units': 'm s-1', 'nodata': -1073741800.0}]}, {'name': 'moisture_content_of_soil_layer', 'description': 'moisture_content_of_soil_layer', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'moisture_content_of_soil_layer', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'moisture_content_of_soil_layer', 'dtype': 'float32', 'units': 'kg m-2', 'nodata': -1073741800.0}]}, {'name': 'snowfall_amount_stratiform_rainfall_rate_0', 'description': 'snowfall_amount_stratiform_rainfall_rate_0', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'snowfall_amount_stratiform_rainfall_rate_0', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'snowfall_amount', 'dtype': 'float32', 'units': 'kg m-2', 'nodata': -1073741800.0}, {'name': 'stratiform_rainfall_rate_0', 'dtype': 'float32', 'units': 'kg m-2 s-1', 'nodata': -1073741800.0}]}, {'name': 'stratiform_rainfall_rate_surface_upward_latent_heat_flux_surface_upward_sensible_heat_flux', 'description': 'stratiform_rainfall_rate_surface_upward_latent_heat_flux_surface_upward_sensible_heat_flux', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'stratiform_rainfall_rate_surface_upward_latent_heat_flux_surface_upward_sensible_heat_flux', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'stratiform_rainfall_rate', 'dtype': 'float32', 'units': 'kg m-2 s-1', 'nodata': -1073741800.0}, {'name': 'surface_upward_latent_heat_flux', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}, {'name': 'surface_upward_sensible_heat_flux', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}]}, {'name': 'surface_downwelling_longwave_flux_surface_downwelling_shortwave_flux_in_air_toa_outgoing_shortwave_flux', 'description': 'surface_downwelling_longwave_flux_surface_downwelling_shortwave_flux_in_air_toa_outgoing_shortwave_flux', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'surface_downwelling_longwave_flux_surface_downwelling_shortwave_flux_in_air_toa_outgoing_shortwave_flux', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'surface_downwelling_longwave_flux', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}, {'name': 'surface_downwelling_shortwave_flux_in_air', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}, {'name': 'toa_outgoing_shortwave_flux', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}]}, {'name': 'toa_outgoing_longwave_flux', 'description': 'toa_outgoing_longwave_flux', 'metadata_type': 'eo', 'metadata': {'platform': {'code': 'mogreps'}, 'product_type': 'toa_outgoing_longwave_flux', 'format': {'name': 'NETCDF'}}, 'measurements': [{'name': 'toa_outgoing_longwave_flux', 'dtype': 'float32', 'units': 'W m-2', 'nodata': -1073741800.0}]}]\n"
]
}
],
"source": [
"def generate_products(nco, dims, variables):\n",
" prod_name = '_'.join(v.name for v in variables)\n",
" return {\n",
" 'name': prod_name,\n",
" 'description': prod_name,\n",
" 'metadata_type': 'eo',\n",
" 'metadata': {\n",
" 'platform': {'code': 'mogreps'},\n",
" 'product_type': prod_name,\n",
" 'format': {\n",
" 'name': 'NETCDF'\n",
" }\n",
" },\n",
" 'measurements': [\n",
" {'name': v.name,\n",
" 'dtype': str(v.dtype),\n",
" 'units': str(v.units),\n",
" 'nodata': v._FillValue}\n",
" for v in variables\n",
" ]\n",
" }\n",
"\n",
"prods = [generate_products(nco, keys, variables)\n",
" for keys, variables in zip(uniquekeys, groups)]\n",
"print(prods)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'description': 'air_pressure_at_sea_level_surface_temperature',\n",
" 'measurements': [{'dtype': 'float32',\n",
" 'name': 'air_pressure_at_sea_level',\n",
" 'nodata': -1073741800.0,\n",
" 'units': 'Pa'},\n",
" {'dtype': 'float32',\n",
" 'name': 'surface_temperature',\n",
" 'nodata': -1073741800.0,\n",
" 'units': 'K'}],\n",
" 'metadata': {'format': {'name': 'NETCDF'},\n",
" 'platform': {'code': 'mogreps'},\n",
" 'product_type': 'air_pressure_at_sea_level_surface_temperature'},\n",
" 'metadata_type': 'eo',\n",
" 'name': 'air_pressure_at_sea_level_surface_temperature'}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prods[1]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Lets make a dataset for:\n",
"```\n",
" (('time', 'latitude', 'longitude'),\n",
" ['air_pressure_at_sea_level', 'surface_temperature']),\n",
" ```"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'bottom': -13.0,\n",
" 'end': numpy.datetime64('2018-02-10T00:00:00.000000000'),\n",
" 'left': 21.519989013671875,\n",
" 'right': 52.0,\n",
" 'start': numpy.datetime64('2018-02-05T00:00:00.000000000'),\n",
" 'top': 17.479999542236328}"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def find_bounds(filename, dims):\n",
" bounds = {}\n",
" with xarray.open_dataset(filename) as ds:\n",
" for dim in dims:\n",
" coord = ds[dim]\n",
" if coord.axis == 'X':\n",
" bounds['left'] = float(coord.min())\n",
" bounds['right'] = float(coord.max())\n",
" elif coord.axis == 'Y':\n",
" bounds['top'] = float(coord.max())\n",
" bounds['bottom'] = float(coord.min())\n",
" elif coord.axis == 'T':\n",
" bounds['start'] = coord.min().data\n",
" bounds['end'] = coord.max().data\n",
" return bounds\n",
"\n",
"bounds = find_bounds(filename, ('time', 'latitude', 'longitude'))\n",
" \n",
"bounds"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2018-02-05T00:00:00.000000000'"
]
},
"execution_count": 46,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"str(bounds['start'])"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'creation_dt': '2018-02-06T01:37:12',\n",
" 'extent': {'coord': {'ll': {'lat': -13.0, 'lon': 21.519989013671875},\n",
" 'lr': {'lat': -13.0, 'lon': 52.0},\n",
" 'ul': {'lat': 17.479999542236328, 'lon': 21.519989013671875},\n",
" 'ur': {'lat': 17.479999542236328, 'lon': 52.0}},\n",
" 'from_dt': '2018-02-05T00:00:00.000000000',\n",
" 'to_dt': '2018-02-10T00:00:00.000000000'},\n",
" 'format': {'name': 'NETCDF'},\n",
" 'id': '82864702-e91a-4dbb-8e40-c9ba4e5d31f3',\n",
" 'image': {'bands': {'air_pressure_at_sea_level': {'layername': 'air_pressure_at_sea_level',\n",
" 'path': '/Users/omad/20180205T0000Z_tropical_africa.nc'},\n",
" 'surface_temperature': {'layername': 'surface_temperature',\n",
" 'path': '/Users/omad/20180205T0000Z_tropical_africa.nc'}}},\n",
" 'lineage': {'source_datasets': {}},\n",
" 'processing_level': 'modelled',\n",
" 'product_type': 'gamma_ray'}"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def make_dataset(filename, dims, variables):\n",
" bounds = find_bounds(filename, dims)\n",
"\n",
" p = Path(filename)\n",
"\n",
" mtime = datetime.fromtimestamp(p.stat().st_mtime)\n",
" return {\n",
" 'id': str(uuid.uuid4()),\n",
" 'processing_level': 'modelled',\n",
" 'product_type': 'gamma_ray',\n",
" 'creation_dt': mtime.isoformat(),\n",
" 'extent': {\n",
" 'coord': {\n",
" 'ul': {'lon': bounds['left'], 'lat': bounds['top']},\n",
" 'ur': {'lon': bounds['right'], 'lat': bounds['top']},\n",
" 'll': {'lon': bounds['left'], 'lat': bounds['bottom']},\n",
" 'lr': {'lon': bounds['right'], 'lat': bounds['bottom']},\n",
" },\n",
" 'from_dt': str(bounds['start']),\n",
" 'to_dt': str(bounds['end']),\n",
" },\n",
" 'format': {'name': 'NETCDF'},\n",
" 'image': {\n",
" 'bands': {\n",
" vname: {\n",
" 'path': filename,\n",
" 'layername': vname,\n",
" } for vname in variables\n",
" }\n",
" },\n",
" 'lineage': {'source_datasets': {}},\n",
" }\n",
"odc_ds = make_dataset(filename, uniquekeys[1], group_names[1])\n",
"odc_ds"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
" name: gamma_ray\n",
" description: The 2015 radiometric or gamma-ray grid of Australia\n",
" metadata_type: eo\n",
"\n",
" metadata:\n",
" platform:\n",
" code: aircraft\n",
" instrument:\n",
" name: gamma_ray spectrometer\n",
" product_type: gamma_ray\n",
" format:\n",
" name: NETCDF\n",
"\n",
" storage:\n",
" crs: GEOGCS[\"GEOCENTRIC DATUM of AUSTRALIA\",DATUM[\"GDA94\",SPHEROID[\"GRS80\",6378137,298.257222101]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433]]\n",
" resolution:\n",
" longitude: 0.001\n",
" latitude: -0.001\n",
"\n",
" measurements:\n",
" - name: rad_air_dose_rate_unfiltered\n",
" dtype: float32\n",
" units: 'nG/h'\n",
" nodata: -99999\n",
" - name: rad_k_equiv_conc_unfiltered\n",
" dtype: float32\n",
" units: '%K'\n",
" nodata: -99999"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([421608.05, 421611. , 421614. , 421617. , 421620. , 421623. ,\n",
" 421626. , 421629. , 421632. , 421635. , 421638. , 421641. ,\n",
" 421644. , 421647. , 421650. , 421653. , 421656. , 421659. ,\n",
" 421662. , 421665. , 421668. , 421671. , 421674. , 421677. ,\n",
" 421680. , 421683. , 421686. , 421689. , 421692. , 421695. ,\n",
" 421698. , 421701. , 421704. , 421707. , 421710. , 421713. ,\n",
" 421716. , 421719. , 421722. , 421725. , 421728. ])"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"nco.variables['time_0'][:]"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"41 [421608. 421611. 421614. 421617. 421620. 421623. 421626. 421629. 421632.\n",
" 421635. 421638. 421641. 421644. 421647. 421650. 421653. 421656. 421659.\n",
" 421662. 421665. 421668. 421671. 421674. 421677. 421680. 421683. 421686.\n",
" 421689. 421692. 421695. 421698. 421701. 421704. 421707. 421710. 421713.\n",
" 421716. 421719. 421722. 421725. 421728.]\n"
]
}
],
"source": [
"def disp_time(name):\n",
" print(len(nco.variables[name][:]), nco.variables[name][:])\n",
"disp_time('time')"
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"41 [421608.05 421611. 421614. 421617. 421620. 421623. 421626.\n",
" 421629. 421632. 421635. 421638. 421641. 421644. 421647.\n",
" 421650. 421653. 421656. 421659. 421662. 421665. 421668.\n",
" 421671. 421674. 421677. 421680. 421683. 421686. 421689.\n",
" 421692. 421695. 421698. 421701. 421704. 421707. 421710.\n",
" 421713. 421716. 421719. 421722. 421725. 421728. ]\n"
]
}
],
"source": [
"disp_time('time_0')"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"40 [421609.5 421612.5 421615.5 421618.5 421621.5 421624.5 421627.5 421630.5\n",
" 421633.5 421636.5 421639.5 421642.5 421645.5 421648.5 421651.5 421654.5\n",
" 421657.5 421660.5 421663.5 421666.5 421669.5 421672.5 421675.5 421678.5\n",
" 421681.5 421684.5 421687.5 421690.5 421693.5 421696.5 421699.5 421702.5\n",
" 421705.5 421708.5 421711.5 421714.5 421717.5 421720.5 421723.5 421726.5]\n"
]
}
],
"source": [
"disp_time('time_1')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Raw
mogreps_prepare.py
# coding=utf-8
"""
Ingest data from the command-line.
Mogreps data from the UK Met Office
"""
from __future__ import absolute_import
import uuid
from datetime import datetime
from itertools import groupby
from operator import attrgetter
from pathlib import Path
import xarray as xr
import click
import netCDF4
import yaml
def find_interesting_vars(nco):
interesting_variables = {vname: v
for vname, v in nco.variables.items()
if 'grid_mapping' in v.ncattrs()}
groups = {}
data = interesting_variables.values()
get_dimensions = attrgetter('dimensions')
data = sorted(data, key=get_dimensions)
for k, g in groupby(data, get_dimensions):
groups[k] = list(g) # Store group iterator as a list
return groups
def generate_product(variables):
prod_name = '_'.join(v.name for v in variables)
return {
'name': prod_name,
'description': prod_name,
'metadata_type': 'eo',
'metadata': {
'platform': {'code': 'mogreps'},
'product_type': prod_name,
'format': {
'name': 'NETCDF'
}
},
'measurements': [
{'name': v.name,
'dtype': str(v.dtype),
'units': str(v.units),
'nodata': float(v._FillValue)}
for v in variables
]
}
def find_bounds(filename, dims):
bounds = {}
with xr.open_dataset(filename) as ds:
for dim in dims:
coord = ds[dim]
if 'axis' not in coord.attrs:
continue
if coord.axis == 'X':
bounds['left'] = float(coord.min())
bounds['right'] = float(coord.max())
elif coord.axis == 'Y':
bounds['top'] = float(coord.max())
bounds['bottom'] = float(coord.min())
elif coord.axis == 'T':
bounds['start'] = coord.min().data
bounds['end'] = coord.max().data
return bounds
def make_dataset(filename, dims, variables):
bounds = find_bounds(filename, dims)
p = Path(filename)
mtime = datetime.fromtimestamp(p.stat().st_mtime)
return {
'id': str(uuid.uuid4()),
'processing_level': 'modelled',
'product_type': 'gamma_ray',
'creation_dt': mtime.isoformat(),
'extent': {
'coord': {
'ul': {'lon': bounds['left'], 'lat': bounds['top']},
'ur': {'lon': bounds['right'], 'lat': bounds['top']},
'll': {'lon': bounds['left'], 'lat': bounds['bottom']},
'lr': {'lon': bounds['right'], 'lat': bounds['bottom']},
},
'from_dt': str(bounds['start']),
'to_dt': str(bounds['end']),
},
'format': {'name': 'NetCDF'},
'image': {
'bands': {
variable.name: {
'path': filename,
'layername': variable.name,
} for variable in variables
}
},
'lineage': {'source_datasets': {}},
}
@click.group()
def cli():
pass
@cli.command(help="Create product definitions from mogreps NetCDF File")
@click.argument('filename', type=click.Path(exists=True, readable=True))
def products(filename):
prods = []
with netCDF4.Dataset(filename) as nco:
groups = find_interesting_vars(nco)
for dims, grouped_vars in groups.items():
prods.append(generate_product(grouped_vars))
print(yaml.safe_dump_all(prods))
@cli.command(help="Create dataset definitions from a mogreps NetCDF File")
@click.argument('filename', type=click.Path(exists=True, readable=True))
def datasets(filename):
datasets = []
with netCDF4.Dataset(filename) as nco:
groups = find_interesting_vars(nco)
for dims, grouped_vars in groups.items():
try:
ds = make_dataset(filename, dims, grouped_vars)
datasets.append(ds)
except Exception as e:
print(e)
print(yaml.safe_dump_all(datasets))
if __name__ == "__main__":
cli()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment