Imputing non-detect data in Python and R

nsqdata_example_subset.csv

ROS-Python.ipynb

ROS-R.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### https://cran.r-project.org/web/packages/NADA/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading required package: survival\n",
      "\n",
      "Attaching package: 'NADA'\n",
      "\n",
      "The following object is masked from 'package:stats':\n",
      "\n",
      "    cor\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>epa_rain_zone</th><th scope=col>season</th><th scope=col>primary_landuse</th><th scope=col>parameter</th><th scope=col>fraction</th><th scope=col>units</th><th scope=col>res</th><th scope=col>qual</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>10</td><td><</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>10</td><td><</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>10</td><td><</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>10</td><td><</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>1</td><td><</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>1</td><td>spring</td><td>Residential</td><td>Cadmium</td><td>Total</td><td>ug/L</td><td>1</td><td><</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       "  & epa_rain_zone & season & primary_landuse & parameter & fraction & units & res & qual\\\\\n",
       "\\hline\n",
       "\t1 & 1 & spring & Residential & Cadmium & Total & ug/L & 10 & <\\\\\n",
       "\t2 & 1 & spring & Residential & Cadmium & Total & ug/L & 10 & <\\\\\n",
       "\t3 & 1 & spring & Residential & Cadmium & Total & ug/L & 10 & <\\\\\n",
       "\t4 & 1 & spring & Residential & Cadmium & Total & ug/L & 10 & <\\\\\n",
       "\t5 & 1 & spring & Residential & Cadmium & Total & ug/L & 1 & <\\\\\n",
       "\t6 & 1 & spring & Residential & Cadmium & Total & ug/L & 1 & <\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "  epa_rain_zone season primary_landuse parameter fraction units res qual\n",
       "1             1 spring     Residential   Cadmium    Total  ug/L  10    <\n",
       "2             1 spring     Residential   Cadmium    Total  ug/L  10    <\n",
       "3             1 spring     Residential   Cadmium    Total  ug/L  10    <\n",
       "4             1 spring     Residential   Cadmium    Total  ug/L  10    <\n",
       "5             1 spring     Residential   Cadmium    Total  ug/L   1    <\n",
       "6             1 spring     Residential   Cadmium    Total  ug/L   1    <"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "library(\"NADA\")\n",
    "\n",
    "main_columns = c(\n",
    "    'epa_rain_zone', 'season', 'primary_landuse',\n",
    "    'parameter', 'fraction', 'units', 'res', 'qual'\n",
    ")\n",
    "df = read.csv(\"nsqdata_example_subset.csv\")[main_columns]\n",
    "head(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the detections limits, compute median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "1"
      ],
      "text/latex": [
       "1"
      ],
      "text/markdown": [
       "1"
      ],
      "text/plain": [
       "[1] 1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "median(df$res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use ROS to impute censored data and show the median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message:\n",
      "In NADA::ros(df$res, df$qual == \"<\"): Dropped censored values that exceed max of uncensored values."
     ]
    },
    {
     "data": {
      "text/html": [
       "0.351893483458947"
      ],
      "text/latex": [
       "0.351893483458947"
      ],
      "text/markdown": [
       "0.351893483458947"
      ],
      "text/plain": [
       "[1] 0.3518935"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfros = NADA::ros(df$res, df$qual=='<')\n",
    "median(dfros)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.1.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}