Skip to content

Instantly share code, notes, and snippets.

@ocefpaf

ocefpaf/PandasStream.ipynb

Created August 9, 2021 19:05
Show Gist options
  • Save ocefpaf/c6c8853df358b20cf1017dcfe1014a7a to your computer and use it in GitHub Desktop.
Save ocefpaf/c6c8853df358b20cf1017dcfe1014a7a to your computer and use it in GitHub Desktop.
Download ZIP
PandasStream.ipynb
Display the source blob
Display the rendered blob
Raw
PandasStream.ipynb
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## Setup the Config"
},
{
"metadata": {
"trusted": true
},
"cell_type": "code",
"source": "from ioos_qc.config import Config\n\n\nconfig = \"\"\"\n contexts:\n - window:\n starting: 2020-01-01T00:00:00Z\n ending: 2020-02-01T00:00:00Z\n streams:\n variable1:\n qartod:\n aggregate:\n gross_range_test:\n suspect_span: [3, 4]\n fail_span: [2, 5]\n rate_of_change_test: \n threshold: [0.01]\n \n spike_test:\n suspect_threshold: [6]\n fail_threshold: [18]\n variable2:\n qartod:\n aggregate:\n gross_range_test:\n suspect_span: [23, 24]\n fail_span: [22, 25]\n - window:\n starting: 2020-02-01T00:00:00Z\n ending: 2020-03-01T00:00:00Z\n \n streams:\n variable1:\n qartod:\n aggregate:\n gross_range_test:\n suspect_span: [43, 44]\n fail_span: [42, 45]\n variable2:\n qartod:\n aggregate:\n gross_range_test:\n suspect_span: [23, 24]\n fail_span: [22, 25]\n\"\"\"\nc = Config(config)\nc.config",
"execution_count": 1,
"outputs": [
{
"output_type": "execute_result",
"execution_count": 1,
"data": {
"text/plain": "OrderedDict([('contexts',\n [{'window': {'starting': datetime.datetime(2020, 1, 1, 0, 0),\n 'ending': datetime.datetime(2020, 2, 1, 0, 0)},\n 'streams': {'variable1': {'qartod': {'aggregate': None,\n 'gross_range_test': {'suspect_span': [3, 4],\n 'fail_span': [2, 5]},\n 'rate_of_change_test': {'threshold': [0.01]},\n 'spike_test': {'suspect_threshold': [6],\n 'fail_threshold': [18]}}},\n 'variable2': {'qartod': {'aggregate': None,\n 'gross_range_test': {'suspect_span': [23, 24],\n 'fail_span': [22, 25]}}}}},\n {'window': {'starting': datetime.datetime(2020, 2, 1, 0, 0),\n 'ending': datetime.datetime(2020, 3, 1, 0, 0)},\n 'streams': {'variable1': {'qartod': {'aggregate': None,\n 'gross_range_test': {'suspect_span': [43, 44],\n 'fail_span': [42, 45]}}},\n 'variable2': {'qartod': {'aggregate': None,\n 'gross_range_test': {'suspect_span': [23, 24],\n 'fail_span': [22, 25]}}}}}])])"
},
"metadata": {}
}
]
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## Setup the sample data"
},
{
"metadata": {
"trusted": true
},
"cell_type": "code",
"source": "import numpy as np\nimport pandas as pd\n\nrows = 50\ntinp = pd.date_range(start='01/01/2020', periods=rows, freq='D').values\ninp1 = np.arange(0, tinp.size)\ninp2 = np.arange(0, tinp.size)\nzinp = np.full_like(tinp, 2.0, dtype=np.float32)\nlat = np.full_like(tinp, 36.1, dtype=np.float32)\nlon = np.full_like(tinp, -76.5, dtype=np.float32)\n\ndf = pd.DataFrame({\n 'time': tinp,\n 'z': zinp,\n 'lat': lat,\n 'lon': lon,\n 'variable1': inp1,\n 'variable2': inp2\n})\ndf",
"execution_count": 2,
"outputs": [
{
"output_type": "execute_result",
"execution_count": 2,
"data": {
"text/plain": " time z lat lon variable1 variable2\n0 2020-01-01 2.0 36.099998 -76.5 0 0\n1 2020-01-02 2.0 36.099998 -76.5 1 1\n2 2020-01-03 2.0 36.099998 -76.5 2 2\n3 2020-01-04 2.0 36.099998 -76.5 3 3\n4 2020-01-05 2.0 36.099998 -76.5 4 4\n5 2020-01-06 2.0 36.099998 -76.5 5 5\n6 2020-01-07 2.0 36.099998 -76.5 6 6\n7 2020-01-08 2.0 36.099998 -76.5 7 7\n8 2020-01-09 2.0 36.099998 -76.5 8 8\n9 2020-01-10 2.0 36.099998 -76.5 9 9\n10 2020-01-11 2.0 36.099998 -76.5 10 10\n11 2020-01-12 2.0 36.099998 -76.5 11 11\n12 2020-01-13 2.0 36.099998 -76.5 12 12\n13 2020-01-14 2.0 36.099998 -76.5 13 13\n14 2020-01-15 2.0 36.099998 -76.5 14 14\n15 2020-01-16 2.0 36.099998 -76.5 15 15\n16 2020-01-17 2.0 36.099998 -76.5 16 16\n17 2020-01-18 2.0 36.099998 -76.5 17 17\n18 2020-01-19 2.0 36.099998 -76.5 18 18\n19 2020-01-20 2.0 36.099998 -76.5 19 19\n20 2020-01-21 2.0 36.099998 -76.5 20 20\n21 2020-01-22 2.0 36.099998 -76.5 21 21\n22 2020-01-23 2.0 36.099998 -76.5 22 22\n23 2020-01-24 2.0 36.099998 -76.5 23 23\n24 2020-01-25 2.0 36.099998 -76.5 24 24\n25 2020-01-26 2.0 36.099998 -76.5 25 25\n26 2020-01-27 2.0 36.099998 -76.5 26 26\n27 2020-01-28 2.0 36.099998 -76.5 27 27\n28 2020-01-29 2.0 36.099998 -76.5 28 28\n29 2020-01-30 2.0 36.099998 -76.5 29 29\n30 2020-01-31 2.0 36.099998 -76.5 30 30\n31 2020-02-01 2.0 36.099998 -76.5 31 31\n32 2020-02-02 2.0 36.099998 -76.5 32 32\n33 2020-02-03 2.0 36.099998 -76.5 33 33\n34 2020-02-04 2.0 36.099998 -76.5 34 34\n35 2020-02-05 2.0 36.099998 -76.5 35 35\n36 2020-02-06 2.0 36.099998 -76.5 36 36\n37 2020-02-07 2.0 36.099998 -76.5 37 37\n38 2020-02-08 2.0 36.099998 -76.5 38 38\n39 2020-02-09 2.0 36.099998 -76.5 39 39\n40 2020-02-10 2.0 36.099998 -76.5 40 40\n41 2020-02-11 2.0 36.099998 -76.5 41 41\n42 2020-02-12 2.0 36.099998 -76.5 42 42\n43 2020-02-13 2.0 36.099998 -76.5 43 43\n44 2020-02-14 2.0 36.099998 -76.5 44 44\n45 2020-02-15 2.0 36.099998 -76.5 45 45\n46 2020-02-16 2.0 36.099998 -76.5 46 46\n47 2020-02-17 2.0 36.099998 -76.5 47 47\n48 2020-02-18 2.0 36.099998 -76.5 48 48\n49 2020-02-19 2.0 36.099998 -76.5 49 49",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>time</th>\n <th>z</th>\n <th>lat</th>\n <th>lon</th>\n <th>variable1</th>\n <th>variable2</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>2020-01-01</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>0</td>\n <td>0</td>\n </tr>\n <tr>\n <th>1</th>\n <td>2020-01-02</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>1</td>\n <td>1</td>\n </tr>\n <tr>\n <th>2</th>\n <td>2020-01-03</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>2</td>\n <td>2</td>\n </tr>\n <tr>\n <th>3</th>\n <td>2020-01-04</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>3</td>\n <td>3</td>\n </tr>\n <tr>\n <th>4</th>\n <td>2020-01-05</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>5</th>\n <td>2020-01-06</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>5</td>\n <td>5</td>\n </tr>\n <tr>\n <th>6</th>\n <td>2020-01-07</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>6</td>\n <td>6</td>\n </tr>\n <tr>\n <th>7</th>\n <td>2020-01-08</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>7</td>\n <td>7</td>\n </tr>\n <tr>\n <th>8</th>\n <td>2020-01-09</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>8</td>\n <td>8</td>\n </tr>\n <tr>\n <th>9</th>\n <td>2020-01-10</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>9</td>\n <td>9</td>\n </tr>\n <tr>\n <th>10</th>\n <td>2020-01-11</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>10</td>\n <td>10</td>\n </tr>\n <tr>\n <th>11</th>\n <td>2020-01-12</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>11</td>\n <td>11</td>\n </tr>\n <tr>\n <th>12</th>\n <td>2020-01-13</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>12</td>\n <td>12</td>\n </tr>\n <tr>\n <th>13</th>\n <td>2020-01-14</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>13</td>\n <td>13</td>\n </tr>\n <tr>\n <th>14</th>\n <td>2020-01-15</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>14</td>\n <td>14</td>\n </tr>\n <tr>\n <th>15</th>\n <td>2020-01-16</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>15</td>\n <td>15</td>\n </tr>\n <tr>\n <th>16</th>\n <td>2020-01-17</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>16</td>\n <td>16</td>\n </tr>\n <tr>\n <th>17</th>\n <td>2020-01-18</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>17</td>\n <td>17</td>\n </tr>\n <tr>\n <th>18</th>\n <td>2020-01-19</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>18</td>\n <td>18</td>\n </tr>\n <tr>\n <th>19</th>\n <td>2020-01-20</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>19</td>\n <td>19</td>\n </tr>\n <tr>\n <th>20</th>\n <td>2020-01-21</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>20</td>\n <td>20</td>\n </tr>\n <tr>\n <th>21</th>\n <td>2020-01-22</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>21</td>\n <td>21</td>\n </tr>\n <tr>\n <th>22</th>\n <td>2020-01-23</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>22</td>\n <td>22</td>\n </tr>\n <tr>\n <th>23</th>\n <td>2020-01-24</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>23</td>\n <td>23</td>\n </tr>\n <tr>\n <th>24</th>\n <td>2020-01-25</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>24</td>\n <td>24</td>\n </tr>\n <tr>\n <th>25</th>\n <td>2020-01-26</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>25</td>\n <td>25</td>\n </tr>\n <tr>\n <th>26</th>\n <td>2020-01-27</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>26</td>\n <td>26</td>\n </tr>\n <tr>\n <th>27</th>\n <td>2020-01-28</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>27</td>\n <td>27</td>\n </tr>\n <tr>\n <th>28</th>\n <td>2020-01-29</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>28</td>\n <td>28</td>\n </tr>\n <tr>\n <th>29</th>\n <td>2020-01-30</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>29</td>\n <td>29</td>\n </tr>\n <tr>\n <th>30</th>\n <td>2020-01-31</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>30</td>\n <td>30</td>\n </tr>\n <tr>\n <th>31</th>\n <td>2020-02-01</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>31</td>\n <td>31</td>\n </tr>\n <tr>\n <th>32</th>\n <td>2020-02-02</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>32</td>\n <td>32</td>\n </tr>\n <tr>\n <th>33</th>\n <td>2020-02-03</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>33</td>\n <td>33</td>\n </tr>\n <tr>\n <th>34</th>\n <td>2020-02-04</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>34</td>\n <td>34</td>\n </tr>\n <tr>\n <th>35</th>\n <td>2020-02-05</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>35</td>\n <td>35</td>\n </tr>\n <tr>\n <th>36</th>\n <td>2020-02-06</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>36</td>\n <td>36</td>\n </tr>\n <tr>\n <th>37</th>\n <td>2020-02-07</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>37</td>\n <td>37</td>\n </tr>\n <tr>\n <th>38</th>\n <td>2020-02-08</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>38</td>\n <td>38</td>\n </tr>\n <tr>\n <th>39</th>\n <td>2020-02-09</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>39</td>\n <td>39</td>\n </tr>\n <tr>\n <th>40</th>\n <td>2020-02-10</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>40</td>\n <td>40</td>\n </tr>\n <tr>\n <th>41</th>\n <td>2020-02-11</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>41</td>\n <td>41</td>\n </tr>\n <tr>\n <th>42</th>\n <td>2020-02-12</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>42</td>\n <td>42</td>\n </tr>\n <tr>\n <th>43</th>\n <td>2020-02-13</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>43</td>\n <td>43</td>\n </tr>\n <tr>\n <th>44</th>\n <td>2020-02-14</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>44</td>\n <td>44</td>\n </tr>\n <tr>\n <th>45</th>\n <td>2020-02-15</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>45</td>\n <td>45</td>\n </tr>\n <tr>\n <th>46</th>\n <td>2020-02-16</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>46</td>\n <td>46</td>\n </tr>\n <tr>\n <th>47</th>\n <td>2020-02-17</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>47</td>\n <td>47</td>\n </tr>\n <tr>\n <th>48</th>\n <td>2020-02-18</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>48</td>\n <td>48</td>\n </tr>\n <tr>\n <th>49</th>\n <td>2020-02-19</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>49</td>\n <td>49</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {}
}
]
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## Setup the NumpyStream"
},
{
"metadata": {
"trusted": true
},
"cell_type": "code",
"source": "from ioos_qc.streams import PandasStream\n\nps = PandasStream(df)\nps",
"execution_count": 3,
"outputs": [
{
"output_type": "execute_result",
"execution_count": 3,
"data": {
"text/plain": "<ioos_qc.streams.PandasStream at 0x7f2d29a732b0>"
},
"metadata": {}
}
]
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## Run the PandasStream through the Config"
},
{
"metadata": {
"trusted": true
},
"cell_type": "code",
"source": "results = ps.run(c)\n",
"execution_count": 4,
"outputs": []
},
{
"metadata": {
"trusted": true,
"scrolled": false
},
"cell_type": "code",
"source": "from ioos_qc.stores import PandasStore\n\n\nstore = PandasStore(\n results,\n axes={\n 't': 'time',\n 'z': 'z',\n 'y': 'lat',\n 'x': 'lon'\n }\n)\n\nstore.compute_aggregate(name='rollup_qc') # Appends to the results internally\nresults_store = store.save(write_data=False, write_axes=False)\ndf = pd.concat([df, results_store], axis=1)\n\ndf",
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"text": "Could not run \"qartod.aggregate: aggregate() missing 1 required positional argument: 'results'\nCould not run \"qartod.aggregate: aggregate() missing 1 required positional argument: 'results'\nCould not run \"qartod.aggregate: aggregate() missing 1 required positional argument: 'results'\nCould not run \"qartod.aggregate: aggregate() missing 1 required positional argument: 'results'\n",
"name": "stderr"
},
{
"output_type": "execute_result",
"execution_count": 5,
"data": {
"text/plain": " time z lat lon variable1 variable2 \\\n0 2020-01-01 2.0 36.099998 -76.5 0 0 \n1 2020-01-02 2.0 36.099998 -76.5 1 1 \n2 2020-01-03 2.0 36.099998 -76.5 2 2 \n3 2020-01-04 2.0 36.099998 -76.5 3 3 \n4 2020-01-05 2.0 36.099998 -76.5 4 4 \n5 2020-01-06 2.0 36.099998 -76.5 5 5 \n6 2020-01-07 2.0 36.099998 -76.5 6 6 \n7 2020-01-08 2.0 36.099998 -76.5 7 7 \n8 2020-01-09 2.0 36.099998 -76.5 8 8 \n9 2020-01-10 2.0 36.099998 -76.5 9 9 \n10 2020-01-11 2.0 36.099998 -76.5 10 10 \n11 2020-01-12 2.0 36.099998 -76.5 11 11 \n12 2020-01-13 2.0 36.099998 -76.5 12 12 \n13 2020-01-14 2.0 36.099998 -76.5 13 13 \n14 2020-01-15 2.0 36.099998 -76.5 14 14 \n15 2020-01-16 2.0 36.099998 -76.5 15 15 \n16 2020-01-17 2.0 36.099998 -76.5 16 16 \n17 2020-01-18 2.0 36.099998 -76.5 17 17 \n18 2020-01-19 2.0 36.099998 -76.5 18 18 \n19 2020-01-20 2.0 36.099998 -76.5 19 19 \n20 2020-01-21 2.0 36.099998 -76.5 20 20 \n21 2020-01-22 2.0 36.099998 -76.5 21 21 \n22 2020-01-23 2.0 36.099998 -76.5 22 22 \n23 2020-01-24 2.0 36.099998 -76.5 23 23 \n24 2020-01-25 2.0 36.099998 -76.5 24 24 \n25 2020-01-26 2.0 36.099998 -76.5 25 25 \n26 2020-01-27 2.0 36.099998 -76.5 26 26 \n27 2020-01-28 2.0 36.099998 -76.5 27 27 \n28 2020-01-29 2.0 36.099998 -76.5 28 28 \n29 2020-01-30 2.0 36.099998 -76.5 29 29 \n30 2020-01-31 2.0 36.099998 -76.5 30 30 \n31 2020-02-01 2.0 36.099998 -76.5 31 31 \n32 2020-02-02 2.0 36.099998 -76.5 32 32 \n33 2020-02-03 2.0 36.099998 -76.5 33 33 \n34 2020-02-04 2.0 36.099998 -76.5 34 34 \n35 2020-02-05 2.0 36.099998 -76.5 35 35 \n36 2020-02-06 2.0 36.099998 -76.5 36 36 \n37 2020-02-07 2.0 36.099998 -76.5 37 37 \n38 2020-02-08 2.0 36.099998 -76.5 38 38 \n39 2020-02-09 2.0 36.099998 -76.5 39 39 \n40 2020-02-10 2.0 36.099998 -76.5 40 40 \n41 2020-02-11 2.0 36.099998 -76.5 41 41 \n42 2020-02-12 2.0 36.099998 -76.5 42 42 \n43 2020-02-13 2.0 36.099998 -76.5 43 43 \n44 2020-02-14 2.0 36.099998 -76.5 44 44 \n45 2020-02-15 2.0 36.099998 -76.5 45 45 \n46 2020-02-16 2.0 36.099998 -76.5 46 46 \n47 2020-02-17 2.0 36.099998 -76.5 47 47 \n48 2020-02-18 2.0 36.099998 -76.5 48 48 \n49 2020-02-19 2.0 36.099998 -76.5 49 49 \n\n variable1_qartod_gross_range_test variable1_qartod_rate_of_change_test \\\n0 4 1.0 \n1 4 1.0 \n2 3 1.0 \n3 1 1.0 \n4 1 1.0 \n5 3 1.0 \n6 4 1.0 \n7 4 1.0 \n8 4 1.0 \n9 4 1.0 \n10 4 1.0 \n11 4 1.0 \n12 4 1.0 \n13 4 1.0 \n14 4 1.0 \n15 4 1.0 \n16 4 1.0 \n17 4 1.0 \n18 4 1.0 \n19 4 1.0 \n20 4 1.0 \n21 4 1.0 \n22 4 1.0 \n23 4 1.0 \n24 4 1.0 \n25 4 1.0 \n26 4 1.0 \n27 4 1.0 \n28 4 1.0 \n29 4 1.0 \n30 4 1.0 \n31 4 NaN \n32 4 NaN \n33 4 NaN \n34 4 NaN \n35 4 NaN \n36 4 NaN \n37 4 NaN \n38 4 NaN \n39 4 NaN \n40 4 NaN \n41 4 NaN \n42 3 NaN \n43 1 NaN \n44 1 NaN \n45 3 NaN \n46 4 NaN \n47 4 NaN \n48 4 NaN \n49 4 NaN \n\n variable1_qartod_spike_test variable2_qartod_gross_range_test \\\n0 2.0 4 \n1 1.0 4 \n2 1.0 4 \n3 1.0 4 \n4 1.0 4 \n5 1.0 4 \n6 1.0 4 \n7 1.0 4 \n8 1.0 4 \n9 1.0 4 \n10 1.0 4 \n11 1.0 4 \n12 1.0 4 \n13 1.0 4 \n14 1.0 4 \n15 1.0 4 \n16 1.0 4 \n17 1.0 4 \n18 1.0 4 \n19 1.0 4 \n20 1.0 4 \n21 1.0 4 \n22 1.0 3 \n23 1.0 1 \n24 1.0 1 \n25 1.0 3 \n26 1.0 4 \n27 1.0 4 \n28 1.0 4 \n29 1.0 4 \n30 2.0 4 \n31 NaN 4 \n32 NaN 4 \n33 NaN 4 \n34 NaN 4 \n35 NaN 4 \n36 NaN 4 \n37 NaN 4 \n38 NaN 4 \n39 NaN 4 \n40 NaN 4 \n41 NaN 4 \n42 NaN 4 \n43 NaN 4 \n44 NaN 4 \n45 NaN 4 \n46 NaN 4 \n47 NaN 4 \n48 NaN 4 \n49 NaN 4 \n\n qartod_rollup_qc \n0 4 \n1 4 \n2 4 \n3 4 \n4 4 \n5 4 \n6 4 \n7 4 \n8 4 \n9 4 \n10 4 \n11 4 \n12 4 \n13 4 \n14 4 \n15 4 \n16 4 \n17 4 \n18 4 \n19 4 \n20 4 \n21 4 \n22 4 \n23 4 \n24 4 \n25 4 \n26 4 \n27 4 \n28 4 \n29 4 \n30 4 \n31 4 \n32 4 \n33 4 \n34 4 \n35 4 \n36 4 \n37 4 \n38 4 \n39 4 \n40 4 \n41 4 \n42 4 \n43 4 \n44 4 \n45 4 \n46 4 \n47 4 \n48 4 \n49 4 ",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>time</th>\n <th>z</th>\n <th>lat</th>\n <th>lon</th>\n <th>variable1</th>\n <th>variable2</th>\n <th>variable1_qartod_gross_range_test</th>\n <th>variable1_qartod_rate_of_change_test</th>\n <th>variable1_qartod_spike_test</th>\n <th>variable2_qartod_gross_range_test</th>\n <th>qartod_rollup_qc</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>2020-01-01</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>0</td>\n <td>0</td>\n <td>4</td>\n <td>1.0</td>\n <td>2.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>1</th>\n <td>2020-01-02</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>1</td>\n <td>1</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>2</th>\n <td>2020-01-03</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>2</td>\n <td>2</td>\n <td>3</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>3</th>\n <td>2020-01-04</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>3</td>\n <td>3</td>\n <td>1</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>4</th>\n <td>2020-01-05</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>4</td>\n <td>4</td>\n <td>1</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>5</th>\n <td>2020-01-06</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>5</td>\n <td>5</td>\n <td>3</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>6</th>\n <td>2020-01-07</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>6</td>\n <td>6</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>7</th>\n <td>2020-01-08</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>7</td>\n <td>7</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>8</th>\n <td>2020-01-09</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>8</td>\n <td>8</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>9</th>\n <td>2020-01-10</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>9</td>\n <td>9</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>10</th>\n <td>2020-01-11</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>10</td>\n <td>10</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>11</th>\n <td>2020-01-12</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>11</td>\n <td>11</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>12</th>\n <td>2020-01-13</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>12</td>\n <td>12</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>13</th>\n <td>2020-01-14</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>13</td>\n <td>13</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>14</th>\n <td>2020-01-15</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>14</td>\n <td>14</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>15</th>\n <td>2020-01-16</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>15</td>\n <td>15</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>16</th>\n <td>2020-01-17</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>16</td>\n <td>16</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>17</th>\n <td>2020-01-18</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>17</td>\n <td>17</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>18</th>\n <td>2020-01-19</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>18</td>\n <td>18</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>19</th>\n <td>2020-01-20</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>19</td>\n <td>19</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>20</th>\n <td>2020-01-21</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>20</td>\n <td>20</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>21</th>\n <td>2020-01-22</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>21</td>\n <td>21</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>22</th>\n <td>2020-01-23</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>22</td>\n <td>22</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>3</td>\n <td>4</td>\n </tr>\n <tr>\n <th>23</th>\n <td>2020-01-24</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>23</td>\n <td>23</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>1</td>\n <td>4</td>\n </tr>\n <tr>\n <th>24</th>\n <td>2020-01-25</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>24</td>\n <td>24</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>1</td>\n <td>4</td>\n </tr>\n <tr>\n <th>25</th>\n <td>2020-01-26</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>25</td>\n <td>25</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>3</td>\n <td>4</td>\n </tr>\n <tr>\n <th>26</th>\n <td>2020-01-27</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>26</td>\n <td>26</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>27</th>\n <td>2020-01-28</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>27</td>\n <td>27</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>28</th>\n <td>2020-01-29</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>28</td>\n <td>28</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>29</th>\n <td>2020-01-30</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>29</td>\n <td>29</td>\n <td>4</td>\n <td>1.0</td>\n <td>1.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>30</th>\n <td>2020-01-31</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>30</td>\n <td>30</td>\n <td>4</td>\n <td>1.0</td>\n <td>2.0</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>31</th>\n <td>2020-02-01</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>31</td>\n <td>31</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>32</th>\n <td>2020-02-02</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>32</td>\n <td>32</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>33</th>\n <td>2020-02-03</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>33</td>\n <td>33</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>34</th>\n <td>2020-02-04</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>34</td>\n <td>34</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>35</th>\n <td>2020-02-05</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>35</td>\n <td>35</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>36</th>\n <td>2020-02-06</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>36</td>\n <td>36</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>37</th>\n <td>2020-02-07</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>37</td>\n <td>37</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>38</th>\n <td>2020-02-08</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>38</td>\n <td>38</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>39</th>\n <td>2020-02-09</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>39</td>\n <td>39</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>40</th>\n <td>2020-02-10</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>40</td>\n <td>40</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>41</th>\n <td>2020-02-11</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>41</td>\n <td>41</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>42</th>\n <td>2020-02-12</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>42</td>\n <td>42</td>\n <td>3</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>43</th>\n <td>2020-02-13</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>43</td>\n <td>43</td>\n <td>1</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>44</th>\n <td>2020-02-14</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>44</td>\n <td>44</td>\n <td>1</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>45</th>\n <td>2020-02-15</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>45</td>\n <td>45</td>\n <td>3</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>46</th>\n <td>2020-02-16</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>46</td>\n <td>46</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>47</th>\n <td>2020-02-17</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>47</td>\n <td>47</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>48</th>\n <td>2020-02-18</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>48</td>\n <td>48</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n <tr>\n <th>49</th>\n <td>2020-02-19</td>\n <td>2.0</td>\n <td>36.099998</td>\n <td>-76.5</td>\n <td>49</td>\n <td>49</td>\n <td>4</td>\n <td>NaN</td>\n <td>NaN</td>\n <td>4</td>\n <td>4</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {}
}
]
}
],
"metadata": {
"language_info": {
"name": "python",
"version": "3.9.6",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3 (ipykernel)",
"language": "python"
},
"gist": {
"id": "",
"data": {
"description": "PandasStream.ipynb",
"public": true
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment