data_access_notebooks/2016-10-12-fetching_data-Copy1.ipynb

parse_coops_stations.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import pandas as pd\nimport pandas_read_xml as pdx\n\n\nroot_key_list = [\"stations\", \"station\"]\nurl = \"https://opendap.co-ops.nos.noaa.gov/stations/stationsXML.jsp\"\n\ndf = pdx.read_xml(\n    url, root_key_list, root_is_rows=False\n).pipe(pdx.fully_flatten)\n\n\n# Fiter out by water level to reduce the data.\ndf = df.loc[df[\"parameter|@name\"] == \"Water Level\"]\ndf = df.drop(\"parameter|@name\", axis=1)\n\n# Cleanup the columns names after the flattening.\nnew_cols = [col.split(\"|\")[-1].strip(\"@\") if \"|\" in col else col.strip(\"@\") for col in df.columns]\ndf.columns = new_cols\n\n# Make the location values numeric so we can filter the data.\ndf[\"lat\"] = pd.to_numeric(df[\"lat\"])\ndf[\"long\"] = pd.to_numeric(df[\"long\"])",
      "execution_count": 1,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "min_lat, max_lat = 24.25, 36.70\nmin_lon, max_lon = -87.40, -74.70\n\nmask = (\n    (df[\"lat\"] >= min_lat) &\n    (df[\"lat\"] <= max_lat) &\n    (df[\"long\"] >= min_lon) &\n    (df[\"long\"] <= max_lon)\n)\n\ndf = df[mask]\n\nprint(f\"We filtered out all stations but those from {', '.join(set(df['state']))} states.\")",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": "We filtered out all stations but those from FL, SC, NC, GA states.\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "df",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 3,
          "data": {
            "text/plain": "                                    name       ID date_established sensorID  \\\n448                                 Duck  8651370       1977-12-01       Y1   \n449                  Oregon Inlet Marina  8652587       1974-08-14       A1   \n460                USCG Station Hatteras  8654467       2010-04-22       Y1   \n466            Beaufort, Duke Marine Lab  8656483       1964-06-04       Y1   \n467                           Wilmington  8658120       1908-01-01       A1   \n472                   Wrightsville Beach  8658163       2004-04-27       A1   \n481                      Springmaid Pier  8661070       1976-09-28       Y1   \n482     Oyster Landing (N Inlet Estuary)  8662245       1982-12-13       A1   \n484    Charleston, Cooper River Entrance  8665530       1899-09-13       A1   \n490                         Fort Pulaski  8670870       1935-07-01       A1   \n501                     Fernandina Beach  8720030       1897-05-08       Y1   \n507            Mayport (Bar Pilots Dock)  8720218       1995-08-01       A1   \n514                          Dames Point  8720219       1977-08-12       A1   \n519  Southbank Riverwalk, St Johns River  8720226       1995-04-26       Y1   \n534                 I-295 Buckman Bridge  8720357       1995-04-25       Y1   \n540         Trident Pier, Port Canaveral  8721604       1994-10-13       Y1   \n541      Lake Worth Pier, Atlantic Ocean  8722670       1970-04-14       A1   \n552                South Port Everglades  8722956       1973-07-20       Y1   \n553           Virginia Key, Biscayne Bay  8723214       1994-01-26       A1   \n564                Vaca Key, Florida Bay  8723970       1970-12-04       Y1   \n570                             Key West  8724580       1913-01-18       Y1   \n571               Naples, Gulf of Mexico  8725110       1965-03-04       A1   \n582                           Fort Myers  8725520       1965-03-08       Y1   \n583                         Port Manatee  8726384       1976-03-24       A1   \n598            St. Petersburg, Tampa Bay  8726520       1946-12-14       Y1   \n603                       Old Port Tampa  8726607       1991-07-17       A1   \n613                             East Bay  8726674       2019-09-09       Y1   \n614                     Clearwater Beach  8726724       1973-04-19       A1   \n620                            Cedar Key  8727520       1914-03-12       A1   \n626                         Apalachicola  8728690       1967-05-01       A1   \n635                          Panama City  8729108       1973-02-11       Y1   \n636                    Panama City Beach  8729210       1989-09-20       A1   \n642                            Pensacola  8729840       1923-04-30       A1   \n\n    DCP status      lat     long state  \n448   1      1  36.1833 -75.7467    NC  \n449   1      1  35.7950 -75.5481    NC  \n460   3      1  35.2086 -75.7042    NC  \n466   3      1  34.7200 -76.6700    NC  \n467   1      1  34.2275 -77.9536    NC  \n472   1      1  34.2133 -77.7867    NC  \n481   1      1  33.6550 -78.9183    SC  \n482   1      0  33.3517 -79.1867    SC  \n484   1      1  32.7808 -79.9236    SC  \n490   1      1  32.0367 -80.9017    GA  \n501   1      1  30.6714 -81.4658    FL  \n507   1      1  30.3982 -81.4279    FL  \n514   1      1  30.3872 -81.5592    FL  \n519   1      1  30.3205 -81.6591    FL  \n534   1      1  30.1924 -81.6900    FL  \n540   1      1  28.4158 -80.5931    FL  \n541   1      1  26.6128 -80.0342    FL  \n552   1      1  26.0817 -80.1167    FL  \n553   1      1  25.7317 -80.1617    FL  \n564   1      1  24.7110 -81.1065    FL  \n570   1      1  24.5508 -81.8081    FL  \n571   1      1  26.1317 -81.8075    FL  \n582   1      1  26.6480 -81.8710    FL  \n583   1      1  27.6383 -82.5625    FL  \n598   1      1  27.7606 -82.6269    FL  \n603   1      1  27.8578 -82.5528    FL  \n613   1      1  27.9231 -82.4214    FL  \n614   1      1  27.9783 -82.8317    FL  \n620   1      1  29.1350 -83.0317    FL  \n626   1      1  29.7244 -84.9806    FL  \n635   1      1  30.1523 -85.6669    FL  \n636   1      1  30.2133 -85.8783    FL  \n642   1      1  30.4044 -87.2112    FL  ",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>name</th>\n      <th>ID</th>\n      <th>date_established</th>\n      <th>sensorID</th>\n      <th>DCP</th>\n      <th>status</th>\n      <th>lat</th>\n      <th>long</th>\n      <th>state</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>448</th>\n      <td>Duck</td>\n      <td>8651370</td>\n      <td>1977-12-01</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>36.1833</td>\n      <td>-75.7467</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>449</th>\n      <td>Oregon Inlet Marina</td>\n      <td>8652587</td>\n      <td>1974-08-14</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>35.7950</td>\n      <td>-75.5481</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>460</th>\n      <td>USCG Station Hatteras</td>\n      <td>8654467</td>\n      <td>2010-04-22</td>\n      <td>Y1</td>\n      <td>3</td>\n      <td>1</td>\n      <td>35.2086</td>\n      <td>-75.7042</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>466</th>\n      <td>Beaufort, Duke Marine Lab</td>\n      <td>8656483</td>\n      <td>1964-06-04</td>\n      <td>Y1</td>\n      <td>3</td>\n      <td>1</td>\n      <td>34.7200</td>\n      <td>-76.6700</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>467</th>\n      <td>Wilmington</td>\n      <td>8658120</td>\n      <td>1908-01-01</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>34.2275</td>\n      <td>-77.9536</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>472</th>\n      <td>Wrightsville Beach</td>\n      <td>8658163</td>\n      <td>2004-04-27</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>34.2133</td>\n      <td>-77.7867</td>\n      <td>NC</td>\n    </tr>\n    <tr>\n      <th>481</th>\n      <td>Springmaid Pier</td>\n      <td>8661070</td>\n      <td>1976-09-28</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>33.6550</td>\n      <td>-78.9183</td>\n      <td>SC</td>\n    </tr>\n    <tr>\n      <th>482</th>\n      <td>Oyster Landing (N Inlet Estuary)</td>\n      <td>8662245</td>\n      <td>1982-12-13</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>33.3517</td>\n      <td>-79.1867</td>\n      <td>SC</td>\n    </tr>\n    <tr>\n      <th>484</th>\n      <td>Charleston, Cooper River Entrance</td>\n      <td>8665530</td>\n      <td>1899-09-13</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>32.7808</td>\n      <td>-79.9236</td>\n      <td>SC</td>\n    </tr>\n    <tr>\n      <th>490</th>\n      <td>Fort Pulaski</td>\n      <td>8670870</td>\n      <td>1935-07-01</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>32.0367</td>\n      <td>-80.9017</td>\n      <td>GA</td>\n    </tr>\n    <tr>\n      <th>501</th>\n      <td>Fernandina Beach</td>\n      <td>8720030</td>\n      <td>1897-05-08</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.6714</td>\n      <td>-81.4658</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>507</th>\n      <td>Mayport (Bar Pilots Dock)</td>\n      <td>8720218</td>\n      <td>1995-08-01</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.3982</td>\n      <td>-81.4279</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>514</th>\n      <td>Dames Point</td>\n      <td>8720219</td>\n      <td>1977-08-12</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.3872</td>\n      <td>-81.5592</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>519</th>\n      <td>Southbank Riverwalk, St Johns River</td>\n      <td>8720226</td>\n      <td>1995-04-26</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.3205</td>\n      <td>-81.6591</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>534</th>\n      <td>I-295 Buckman Bridge</td>\n      <td>8720357</td>\n      <td>1995-04-25</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.1924</td>\n      <td>-81.6900</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>540</th>\n      <td>Trident Pier, Port Canaveral</td>\n      <td>8721604</td>\n      <td>1994-10-13</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>28.4158</td>\n      <td>-80.5931</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>541</th>\n      <td>Lake Worth Pier, Atlantic Ocean</td>\n      <td>8722670</td>\n      <td>1970-04-14</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>26.6128</td>\n      <td>-80.0342</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>552</th>\n      <td>South Port Everglades</td>\n      <td>8722956</td>\n      <td>1973-07-20</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>26.0817</td>\n      <td>-80.1167</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>553</th>\n      <td>Virginia Key, Biscayne Bay</td>\n      <td>8723214</td>\n      <td>1994-01-26</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>25.7317</td>\n      <td>-80.1617</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>564</th>\n      <td>Vaca Key, Florida Bay</td>\n      <td>8723970</td>\n      <td>1970-12-04</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>24.7110</td>\n      <td>-81.1065</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>570</th>\n      <td>Key West</td>\n      <td>8724580</td>\n      <td>1913-01-18</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>24.5508</td>\n      <td>-81.8081</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>571</th>\n      <td>Naples, Gulf of Mexico</td>\n      <td>8725110</td>\n      <td>1965-03-04</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>26.1317</td>\n      <td>-81.8075</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>582</th>\n      <td>Fort Myers</td>\n      <td>8725520</td>\n      <td>1965-03-08</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>26.6480</td>\n      <td>-81.8710</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>583</th>\n      <td>Port Manatee</td>\n      <td>8726384</td>\n      <td>1976-03-24</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>27.6383</td>\n      <td>-82.5625</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>598</th>\n      <td>St. Petersburg, Tampa Bay</td>\n      <td>8726520</td>\n      <td>1946-12-14</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>27.7606</td>\n      <td>-82.6269</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>603</th>\n      <td>Old Port Tampa</td>\n      <td>8726607</td>\n      <td>1991-07-17</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>27.8578</td>\n      <td>-82.5528</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>613</th>\n      <td>East Bay</td>\n      <td>8726674</td>\n      <td>2019-09-09</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>27.9231</td>\n      <td>-82.4214</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>614</th>\n      <td>Clearwater Beach</td>\n      <td>8726724</td>\n      <td>1973-04-19</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>27.9783</td>\n      <td>-82.8317</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>620</th>\n      <td>Cedar Key</td>\n      <td>8727520</td>\n      <td>1914-03-12</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>29.1350</td>\n      <td>-83.0317</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>626</th>\n      <td>Apalachicola</td>\n      <td>8728690</td>\n      <td>1967-05-01</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>29.7244</td>\n      <td>-84.9806</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>635</th>\n      <td>Panama City</td>\n      <td>8729108</td>\n      <td>1973-02-11</td>\n      <td>Y1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.1523</td>\n      <td>-85.6669</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>636</th>\n      <td>Panama City Beach</td>\n      <td>8729210</td>\n      <td>1989-09-20</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.2133</td>\n      <td>-85.8783</td>\n      <td>FL</td>\n    </tr>\n    <tr>\n      <th>642</th>\n      <td>Pensacola</td>\n      <td>8729840</td>\n      <td>1923-04-30</td>\n      <td>A1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>30.4044</td>\n      <td>-87.2112</td>\n      <td>FL</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
          },
          "metadata": {}
        }
      ]
    }
  ],
  "metadata": {
    "anaconda-cloud": {},
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3 (ipykernel)",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.4",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "",
      "data": {
        "description": "data_access_notebooks/2016-10-12-fetching_data-Copy1.ipynb",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}