Parse GRIMM particle counts and estimate mass concentrations

read_grimm.py

#!/usr/bin/env python3

import numpy as np
import os
import pandas as pd

DATA_PATH = '/home/data/alta_udot/grimm_1109/'

BIN_SIZE_UPPER = np.array([0.25,
                           0.28,
                           0.3,
                           0.35,
                           0.4,
                           0.45,
                           0.5,
                           0.58,
                           0.65,
                           0.7,
                           0.8,
                           1.0,
                           1.3,
                           1.6,
                           2,
                           2.5,
                           3,
                           3.5,
                           4,
                           5,
                           6.5,
                           7.5,
                           8.5,
                           10,
                           12.5,
                           15,
                           17.5,
                           20,
                           25,
                           30,
                           32,
                           9999]) * 1e-6  # [m]

PARTICLE_DENSITY = 2.5e12  # [ug m-3]

TRANSMISSION_EFFICIENCY = np.array([0.9984480257,
                                    0.9981661124,
                                    0.9979658865,
                                    0.9974219954,
                                    0.9968158814,
                                    0.9961473774,
                                    0.9954165081,
                                    0.9941177784,
                                    0.992851444,
                                    0.991873087,
                                    0.9897330241,
                                    0.9847273933,
                                    0.9754386725,
                                    0.9640684329,
                                    0.9457855443,
                                    0.9181626064,
                                    0.8856016482,
                                    0.8485313359,
                                    0.8074187886,
                                    0.7150943161,
                                    0.5595642092,
                                    0.451205686,
                                    0.3445254765,
                                    0.1974961192,
                                    0.0234574173,
                                    0,
                                    0,
                                    0,
                                    0,
                                    0,
                                    0,
                                    0])  # []

def read_grimm_mc(ndays=5):
    # Last 5 days of particle count distribution data
    files = sorted(os.listdir(DATA_PATH))[-ndays:]

    data_list = []
    for f in files:
        df = pd.read_csv(os.path.join(DATA_PATH, f),
                        header=None,
                        index_col=0,
                        parse_dates=True)
        data_list.append(df)

    data = pd.concat(data_list)
    data.index.name = 'epochtime'
    data.tz_localize('UTC')
    data = data / 1e-4  # transform reported [count/100mL] to [count m-3]

    # Time average [6 second] observations to [1 minute]
    data = data.resample('1Min').mean()

    # Correct for inlet transmission efficiency
    data = data / TRANSMISSION_EFFICIENCY

    # Calculate mass concentration from bin size average and density estimate
    bin_size_mean = (BIN_SIZE_UPPER[1:] + BIN_SIZE_UPPER[:-1]) / 2
    bin_size_mean = np.append(bin_size_mean, 9999)
    p_volume = (4/3) * np.pi * (bin_size_mean / 2)**3  # [m3]
    p_mass = p_volume * PARTICLE_DENSITY  # [ug]
    mass_conc = p_mass * data

    # Sum particle mass concentrations within <2.5um size range
    return pd.DataFrame({
        'pm2_5': mass_conc[mass_conc.columns[bin_size_mean <= 2.5 * 1e-6]].sum(axis=1),
        'pm10': mass_conc[mass_conc.columns[bin_size_mean <= 10 * 1e-6]].sum(axis=1)
    })
  

if __name__ == '__main__':
    print(read_grimm())

read_grimm.r

# Read GRIMM data and calculate PM2.5 and PM10 mass concentrations
# Ben Fasoli

library(tidyverse)
library(uataq) # https://github.com/benfasoli/uataq

# Read and parse dataset
files <- dir('/home/rosspetersen/alta', full.names = T)
txt <- do.call(c, lapply(files, readLines, skipNul = T))
df <- uataq::breakstr(txt)
colnames(df) <- c('Time_UTC', 
                  paste0('S', stringr::str_pad(1:(ncol(df)-1), 2, pad = '0')))
df$Time_UTC <- as.POSIXct(df$Time_UTC, tz = 'UTC')
attributes(df$Time_UTC)$tzone <- 'America/Denver'

# Particles reported as #/100mL
# Use bin sizes to estimate mass concentrations
count <- df[, 2:ncol(df)] / 1e-4 # C m-3

rho <- 2.5e12 # ug m-3
bins_um <- c(0.25, 0.28, 0.3, 0.35, 0.4, 0.45, 0.5, 0.58, 0.65, 0.7, 0.8, 1.0, 1.3, 1.6, 2, 
             2.5, 3, 3.5, 4, 5, 6.5, 7.5, 8.5, 10, 12.5, 15, 17.5, 20, 25, 30, 32, Inf) # um
bins_m <- bins_um * 1e-6 # m

particle_volume <- (4/3) * pi * (bins_m / 2)^3 # m3
particle_mass <- particle_volume * rho # ug
particle_mass_mat <- matrix(particle_mass, # ug
                            nrow = nrow(count), 
                            ncol = ncol(count), 
                            byrow = T)
mc <- particle_mass_mat * count # ug m-3

conc <- data_frame(
  Time_UTC = df$Time_UTC,
  PM2.5 = rowSums(mc[, bins_um <= 2.5], na.rm = T),
  PM10 = rowSums(mc[, bins_um <= 10], na.rm = T)
)