ISO Harvesting

harvest.py

#!python
# coding=utf-8
import os
from thredds_crawler.crawl import Crawl
from thredds_iso_harvester.harvest import ThreddsIsoHarvester

SAVE_DIR = "/srv/http/iso"
LOG_DIR = "/var/log/iso_harvest"

# AOOS
ThreddsIsoHarvester(catalog_url="http://thredds.axiomdatascience.com/thredds/catalogs/aoos.html",
    out_dir=SAVE_DIR + "/aoos",
    log_file=LOG_DIR + "/aoos.log")

# CeNCOOS
ThreddsIsoHarvester(catalog_url="http://thredds.axiomdatascience.com/thredds/catalogs/cencoos.html",
    out_dir=SAVE_DIR + "/cencoos",
    log_file=LOG_DIR + "/cencoos.log")

# SECOORA
ThreddsIsoHarvester(catalog_url="http://thredds.axiomdatascience.com/thredds/catalogs/secoora.xml",
    out_dir=SAVE_DIR + "/secoora",
    log_file=LOG_DIR + "/secoora.log")

# UNIDATA
unidata_selects = [".*Best.*"]
unidata_skips   = Crawl.SKIPS  + [".*grib2", ".*grib1", ".*GrbF.*", ".*ncx2",
                                  "Radar Data", "Station Data",
                                  "Point Feature Collections", "Satellite Data",
                                  "Unidata NEXRAD Composites \(GINI\)",
                                  "Unidata case studies",
                                  ".*Reflectivity-[0-9]{8}"]
ThreddsIsoHarvester(catalog_url="http://thredds.ucar.edu/thredds/catalog.html",
    out_dir=SAVE_DIR + "/unidata",
    log_file=LOG_DIR + "/unidata.log",
    select=unidata_selects,
    skip=unidata_skips)

# NDBC 'stdmet'
ndbc_stdmet_selects = [".+\.ncml"]
ThreddsIsoHarvester(catalog_url="http://dods.ndbc.noaa.gov/thredds/catalog/data/stdmet/catalog.xml",
    out_dir=SAVE_DIR + "/ndbc_stdmet",
    log_file=LOG_DIR + "/ndbc_stdmet.log",
    select=ndbc_stdmet_selects)

# NANOOS OSU
ThreddsIsoHarvester(catalog_url="http://ona.coas.oregonstate.edu:8080/thredds/catalog.xml",
    out_dir=SAVE_DIR + "/nanoos_osu",
    log_file=LOG_DIR + "/nanoos_osu.log")