Created
August 21, 2018 22:52
-
-
Save JiaweiZhuang/2e538530cbeed20930f5d737c6e46444 to your computer and use it in GitHub Desktop.
Extract a group in a NetCDF4 file as a standalone NC file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "netcdf var_in_group {\r\n", | |
| "\r\n", | |
| "// global attributes:\r\n", | |
| "\t\t:Conventions = \"CF-1.7\" ;\r\n", | |
| "\t\t:institution = \"KNMI\" ;\r\n", | |
| "\t\t:source = \"Sentinel 5 precursor, TROPOMI, space-borne remote sensing, L2\" ;\r\n", | |
| "\t\t:summary = \"TROPOMI/S5P NO2 1-Orbit L2 Swath 7x3.5km\" ;\r\n", | |
| "\t\t:tracking_id = \"b9374c56-17fe-4850-871f-1e47d52317bf\" ;\r\n", | |
| "\t\t:id = \"S5P_RPRO_L2__NO2____20180401T161408_20180401T175736_02416_01_010001_20180517T055107\" ;\r\n", | |
| "\t\t:time_reference = \"2018-04-01T00:00:00Z\" ;\r\n", | |
| "\t\t:time_reference_days_since_1950 = 24927 ;\r\n", | |
| "\t\t:time_reference_julian_day = 2458209.5 ;\r\n", | |
| "\t\t:time_reference_seconds_since_1970 = 1522540800LL ;\r\n", | |
| "\t\t:time_coverage_start = \"2018-04-01T16:36:42Z\" ;\r\n", | |
| "\t\t:time_coverage_end = \"2018-04-01T17:35:06Z\" ;\r\n", | |
| "\t\t:time_coverage_duration = \"PT3504.000S\" ;\r\n", | |
| "\t\t:time_coverage_resolution = \"PT1.080S\" ;\r\n", | |
| "\t\t:orbit = 2416 ;\r\n", | |
| "\t\t:references = \"http://www.tropomi.eu/data-products/nitrogen-dioxide\" ;\r\n", | |
| "\t\t:processor_version = \"1.0.1\" ;\r\n", | |
| "\t\t:keywords_vocabulary = \"AGU index terms, http://publications.agu.org/author-resource-center/index-terms/\" ;\r\n", | |
| "\t\t:keywords = \"0345 Pollution, Urban and Regional; 0365 Troposphere, Composition and Chemistry; 0368 Troposphere, Constituent Transport and Chemistry; 3360 Remote Sensing; 3363 Stratospheric Dynamics\" ;\r\n", | |
| "\t\t:standard_name_vocabulary = \"NetCDF Climate and Forecast Metadata Conventions Standard Name Table (v29, 08 July 2015), http://cfconventions.org/standard-names.html\" ;\r\n", | |
| "\t\t:naming_authority = \"nl.knmi\" ;\r\n", | |
| "\t\t:cdm_data_type = \"Swath\" ;\r\n", | |
| "\t\t:date_created = \"2018-05-17T05:52:04Z\" ;\r\n", | |
| "\t\t:creator_name = \"The Sentinel 5 Precursor TROPOMI Level 2 products are developed with funding from the European Space Agency (ESA), the Netherlands Space Office (NSO), the Belgian Science Policy Office, the German Aerospace Center (DLR) and the Bayerisches Staatsministerium für Wirtschaft und Medien, Energie und Technologie (StMWi).\" ;\r\n", | |
| "\t\t:creator_url = \"http://www.tropomi.eu\" ;\r\n", | |
| "\t\t:creator_email = \"[email protected]\" ;\r\n", | |
| "\t\t:project = \"Sentinel 5 precursor/TROPOMI\" ;\r\n", | |
| "\t\t:geospatial_lat_min = -89.97005f ;\r\n", | |
| "\t\t:geospatial_lat_max = 89.97588f ;\r\n", | |
| "\t\t:geospatial_lon_min = 179.9999f ;\r\n", | |
| "\t\t:geospatial_lon_max = -180.f ;\r\n", | |
| "\t\t:license = \"No conditions apply\" ;\r\n", | |
| "\t\t:platform = \"S5P\" ;\r\n", | |
| "\t\t:sensor = \"TROPOMI\" ;\r\n", | |
| "\t\t:spatial_resolution = \"7x3.5km2\" ;\r\n", | |
| "\t\t:cpp_compiler_version = \"g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\" ;\r\n", | |
| "\t\t:cpp_compiler_flags = \"-g -O2 -fPIC -std=c++11 -W -Wall -Wno-ignored-qualifiers -Wno-write-strings -Wno-unused-variable -DTROPNLL2DP\" ;\r\n", | |
| "\t\t:f90_compiler_version = \"GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\" ;\r\n", | |
| "\t\t:f90_compiler_flags = \"-gdwarf-3 -O2 -fPIC -cpp -ffpe-trap=invalid -fno-range-check -frecursive -fimplicit-none -ffree-line-length-none -DTROPNLL2DP -Wuninitialized -Wtabs\" ;\r\n", | |
| "\t\t:build_date = \"2018-05-03T16:00:00Z\" ;\r\n", | |
| "\t\t:revision_control_identifier = \"15d6ea7e0ee7\" ;\r\n", | |
| "\t\t:geolocation_grid_from_band = 4 ;\r\n", | |
| "\t\t:identifier_product_doi = \"10.5270/S5P-s4ljg54\" ;\r\n", | |
| "\t\t:identifier_product_doi_authority = \"http://dx.doi.org/\" ;\r\n", | |
| "\t\t:algorithm_version = \"1.0.0\" ;\r\n", | |
| "\t\t:title = \"TROPOMI/S5P NO2 1-Orbit L2 Swath 7x3.5km\" ;\r\n", | |
| "\t\t:product_version = \"1.0.0\" ;\r\n", | |
| "\t\t:Status_MET_2D = \"Nominal\" ;\r\n", | |
| "\t\t:Status_NISE__ = \"Nominal\" ;\r\n", | |
| "\t\t:Status_CTMFCT = \"Nominal\" ;\r\n", | |
| "\t\t:history = \"Tue Aug 14 20:45:02 2018: ncks -C -v latitude,longitude,time_utc,air_mass_factor_troposphere,nitrogendioxide_tropospheric_column S5P_RPRO_L2__NO2____20180401T161408_20180401T175736_02416_01_010001_20180517T055107.nc ./post_process/S5P_RPRO_L2__NO2____20180401T161408_20180401T175736_02416_01_010001_20180517T055107.nc\\n2018-05-17 06:48:55 f_s5pops tropnll2dp /mnt/data1/storage_offl_l2/cache_offl_l2/WORKING-563965794/JobOrder.563958975.xml; 2018-06-14 17:14:18 TM5-MP-DOMINO offline\" ;\r\n", | |
| "\t\t:processing_status = \"OFFL-processing nominal product\" ;\r\n", | |
| "\t\t:date_modified = \"2018-06-14T17:14:18Z\" ;\r\n", | |
| "\t\t:NCO = \"\\\"4.6.4\\\"\" ;\r\n", | |
| "\r\n", | |
| "group: PRODUCT {\r\n", | |
| " dimensions:\r\n", | |
| " \ttime = 1 ;\r\n", | |
| " \tscanline = 3245 ;\r\n", | |
| " \tground_pixel = 450 ;\r\n", | |
| " variables:\r\n", | |
| " \tfloat air_mass_factor_troposphere(time, scanline, ground_pixel) ;\r\n", | |
| " \t\tair_mass_factor_troposphere:units = \"1\" ;\r\n", | |
| " \t\tair_mass_factor_troposphere:long_name = \"Tropospheric air mass factor\" ;\r\n", | |
| " \t\tair_mass_factor_troposphere:coordinates = \"/PRODUCT/longitude /PRODUCT/latitude\" ;\r\n", | |
| " \t\tair_mass_factor_troposphere:ancillary_variables = \"tm5_tropopause_layer_index\" ;\r\n", | |
| " \t\tair_mass_factor_troposphere:_FillValue = 9.96921e+36f ;\r\n", | |
| " \tfloat latitude(time, scanline, ground_pixel) ;\r\n", | |
| " \t\tlatitude:long_name = \"pixel center latitude\" ;\r\n", | |
| " \t\tlatitude:units = \"degrees_north\" ;\r\n", | |
| " \t\tlatitude:standard_name = \"latitude\" ;\r\n", | |
| " \t\tlatitude:valid_min = -90.f ;\r\n", | |
| " \t\tlatitude:valid_max = 90.f ;\r\n", | |
| " \t\tlatitude:bounds = \"/PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds\" ;\r\n", | |
| " \t\tlatitude:_FillValue = 9.96921e+36f ;\r\n", | |
| " \tfloat longitude(time, scanline, ground_pixel) ;\r\n", | |
| " \t\tlongitude:long_name = \"pixel center longitude\" ;\r\n", | |
| " \t\tlongitude:units = \"degrees_east\" ;\r\n", | |
| " \t\tlongitude:standard_name = \"longitude\" ;\r\n", | |
| " \t\tlongitude:valid_min = -180.f ;\r\n", | |
| " \t\tlongitude:valid_max = 180.f ;\r\n", | |
| " \t\tlongitude:bounds = \"/PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds\" ;\r\n", | |
| " \t\tlongitude:_FillValue = 9.96921e+36f ;\r\n", | |
| " \tfloat nitrogendioxide_tropospheric_column(time, scanline, ground_pixel) ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:units = \"mol m-2\" ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:standard_name = \"troposphere_mole_content_of_nitrogen_dioxide\" ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:long_name = \"Tropospheric vertical column of nitrogen dioxide\" ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:coordinates = \"longitude latitude\" ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:ancillary_variables = \"nitrogendioxide_tropospheric_column_precision air_mass_factor_troposphere air_mass_factor_total averaging_kernel\" ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:multiplication_factor_to_convert_to_molecules_percm2 = 6.02214e+19f ;\r\n", | |
| " \t\tnitrogendioxide_tropospheric_column:_FillValue = 9.96921e+36f ;\r\n", | |
| " \tstring time_utc(time, scanline) ;\r\n", | |
| " \t\ttime_utc:long_name = \"Time of observation as ISO 8601 date-time string\" ;\r\n", | |
| " \t\tstring time_utc:_FillValue = \"\" ;\r\n", | |
| " } // group PRODUCT\r\n", | |
| "}\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!ncdump -h var_in_group.nc # NC file with groups" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import xarray as xr" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "<xarray.Dataset>\n", | |
| "Dimensions: (ground_pixel: 450, scanline: 3245, time: 1)\n", | |
| "Coordinates:\n", | |
| " latitude (time, scanline, ground_pixel) float32 ...\n", | |
| " longitude (time, scanline, ground_pixel) float32 ...\n", | |
| "Dimensions without coordinates: ground_pixel, scanline, time\n", | |
| "Data variables:\n", | |
| " air_mass_factor_troposphere (time, scanline, ground_pixel) float32 ...\n", | |
| " nitrogendioxide_tropospheric_column (time, scanline, ground_pixel) float32 ...\n", | |
| " time_utc (time, scanline) object ..." | |
| ] | |
| }, | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "ds = xr.open_dataset('var_in_group.nc', group='PRODUCT')\n", | |
| "ds" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "''" | |
| ] | |
| }, | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "# Fix attribute bug\n", | |
| "# See https://stackoverflow.com/questions/50475453/xarray-cannot-serialize-coordinates\n", | |
| "del ds['air_mass_factor_troposphere'].attrs['coordinates']\n", | |
| "\n", | |
| "# Fix filled value bug\n", | |
| "# See https://github.com/pydata/xarray/pull/1802\n", | |
| "ds['time_utc'].encoding.pop('_FillValue')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "ds.to_netcdf('extracted.nc')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "netcdf extracted {\r\n", | |
| "dimensions:\r\n", | |
| "\ttime = 1 ;\r\n", | |
| "\tscanline = 3245 ;\r\n", | |
| "\tground_pixel = 450 ;\r\n", | |
| "variables:\r\n", | |
| "\tfloat air_mass_factor_troposphere(time, scanline, ground_pixel) ;\r\n", | |
| "\t\tair_mass_factor_troposphere:_FillValue = 9.96921e+36f ;\r\n", | |
| "\t\tair_mass_factor_troposphere:units = \"1\" ;\r\n", | |
| "\t\tair_mass_factor_troposphere:long_name = \"Tropospheric air mass factor\" ;\r\n", | |
| "\t\tair_mass_factor_troposphere:ancillary_variables = \"tm5_tropopause_layer_index\" ;\r\n", | |
| "\t\tair_mass_factor_troposphere:coordinates = \"latitude longitude\" ;\r\n", | |
| "\tfloat latitude(time, scanline, ground_pixel) ;\r\n", | |
| "\t\tlatitude:_FillValue = 9.96921e+36f ;\r\n", | |
| "\t\tlatitude:long_name = \"pixel center latitude\" ;\r\n", | |
| "\t\tlatitude:units = \"degrees_north\" ;\r\n", | |
| "\t\tlatitude:standard_name = \"latitude\" ;\r\n", | |
| "\t\tlatitude:valid_min = -90.f ;\r\n", | |
| "\t\tlatitude:valid_max = 90.f ;\r\n", | |
| "\t\tlatitude:bounds = \"/PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds\" ;\r\n", | |
| "\tfloat longitude(time, scanline, ground_pixel) ;\r\n", | |
| "\t\tlongitude:_FillValue = 9.96921e+36f ;\r\n", | |
| "\t\tlongitude:long_name = \"pixel center longitude\" ;\r\n", | |
| "\t\tlongitude:units = \"degrees_east\" ;\r\n", | |
| "\t\tlongitude:standard_name = \"longitude\" ;\r\n", | |
| "\t\tlongitude:valid_min = -180.f ;\r\n", | |
| "\t\tlongitude:valid_max = 180.f ;\r\n", | |
| "\t\tlongitude:bounds = \"/PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds\" ;\r\n", | |
| "\tfloat nitrogendioxide_tropospheric_column(time, scanline, ground_pixel) ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:_FillValue = 9.96921e+36f ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:units = \"mol m-2\" ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:standard_name = \"troposphere_mole_content_of_nitrogen_dioxide\" ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:long_name = \"Tropospheric vertical column of nitrogen dioxide\" ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:ancillary_variables = \"nitrogendioxide_tropospheric_column_precision air_mass_factor_troposphere air_mass_factor_total averaging_kernel\" ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:multiplication_factor_to_convert_to_molecules_percm2 = 6.02214e+19f ;\r\n", | |
| "\t\tnitrogendioxide_tropospheric_column:coordinates = \"latitude longitude\" ;\r\n", | |
| "\tstring time_utc(time, scanline) ;\r\n", | |
| "\t\ttime_utc:long_name = \"Time of observation as ISO 8601 date-time string\" ;\r\n", | |
| "}\r\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "!ncdump -h extracted.nc # NC file without groups" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.6.6" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment