Obtain data from MOJAVE

get_mojave_uvfits.py

import os
import urllib
import bs4 as BeautifulSoup
import fnmatch


mojave_multifreq_url = "http://www.cv.nrao.edu/2cmVLBA/data/multifreq/"
mojave_u_url = "http://www.cv.nrao.edu/2cmVLBA/data/"
mojave_l_url = "http://www.cv.nrao.edu/MOJAVELBAND"

download_dir = '/home/ilya/code/vlbi_errors/examples/mojave'
mojave_bands = ['x', 'y', 'j', 'u', 'l18', 'l20', 'l21', 'l22']
l_bands = ['l18', 'l20', 'l21', 'l22']


def mojave_uv_fits_fname(source, band, epoch, ext='uvf'):
    return source + '.' + band + '.' + epoch + '.' + ext


def download_mojave_uv_fits(source, epochs=None, bands=None, download_dir=None):
    """
    Download FITS-files with self-calibrated uv-data from MOJAVE server.

    :param source:
        Source name [B1950].
    :param epochs: (optional)
        Iterable of epochs to download [YYYY-MM-DD]. If ``None`` then download
        all. (default: ``None``)
    :param bands: (optional)
        Iterable bands to download ('x', 'y', 'j' or 'u'). If ``None`` then
        download all available bands for given epochs. (default: ``None``)
    :param download_dir: (optional)
        Local directory to save files. If ``None`` then use CWD. (default:
        ``None``)
    """
    if bands is None:
        bands = mojave_bands
    else:
        assert set(bands).issubset(mojave_bands)

    if 'u' in bands:
        # Finding epochs in u-band data
        request = urllib.request.Request(os.path.join(mojave_u_url, source))
        response = urllib.request.urlopen(request)
        soup = BeautifulSoup.BeautifulSoup(response)

        available_epochs = list()
        for a in soup.findAll('a'):
            if fnmatch.fnmatch(a['href'], "*_*_*"):
                epoch = str(a['href'].strip('/'))
                available_epochs.append(epoch)

        if epochs is not None:
            if not set(epochs).issubset(available_epochs):
                raise Exception(" No epochs {} in MOJAVE data."
                                " Available are {}".format(epochs,
                                                           available_epochs))
        else:
            epochs = available_epochs

        # Downloading u-band data
        u_url = os.path.join(mojave_u_url, source)
        for epoch in epochs:
            fname = mojave_uv_fits_fname(source, 'u', epoch)
            url = os.path.join(u_url, epoch, fname)
            print("Downloading file {}".format(fname))
            path = os.path.join(download_dir, fname)
            if os.path.isfile(path):
                print("File {} does exist in {}."
                      " Skipping...".format(fname, download_dir))
                continue
            urllib.request.urlretrieve(url, path)

    # Downloading (optionally) x, y & j-band data
    request = urllib.request.Request(mojave_multifreq_url)
    response = urllib.request.urlopen(request)
    soup = BeautifulSoup.BeautifulSoup(response)

    download_list = list()
    for a in soup.findAll('a'):
        if source in a['href'] and '.uvf' in a['href']:
            fname = a['href']
            epoch = fname.split('.')[2]
            band = fname.split('.')[1]
            if band in bands:
                if epochs is None:
                    download_list.append(os.path.join(mojave_multifreq_url,
                                                      fname))
                else:
                    if epoch in epochs:
                        download_list.append(os.path.join(mojave_multifreq_url,
                                                          fname))
    for url in download_list:
        fname = os.path.split(url)[-1]
        print("Downloading file {}".format(fname))
        path = os.path.join(download_dir, fname)
        if os.path.isfile(path):
            print("File {} does exist in {}."
                  " Skipping...".format(fname, download_dir))
            continue
        urllib.request.urlretrieve(url, os.path.join(download_dir, fname))

    # Downloading (optionally) l-band data
    if 'l18' in bands or 'l20' in bands or 'l21' in bands or 'l22' in bands:
        request = urllib.request.Request(os.path.join(mojave_l_url, source))
        try:
            response = urllib.request.urlopen(request)
        except urllib.error.HTTPError:
            print("No L-bands data available")
            return
        soup = BeautifulSoup.BeautifulSoup(response)

        available_epochs = list()
        for a in soup.findAll('a'):
            if fnmatch.fnmatch(a['href'], "*_*_*"):
                epoch = str(a['href'].strip('/'))
                available_epochs.append(epoch)

        if epochs is not None:
            if not set(epochs).issubset(available_epochs):
                raise Exception(" No epochs {} in MOJAVE data")
        else:
            epochs = available_epochs

        # Downloading l-band data
        l_url = os.path.join(mojave_l_url, source)
        for epoch in epochs:
            for band in bands:
                if band in l_bands:
                    fname = mojave_uv_fits_fname(source, band, epoch)
                    url = os.path.join(l_url, epoch, fname)
                    print("Downloading file {}".format(fname))
                    path = os.path.join(download_dir, fname)
                    if os.path.isfile(path):
                        print("File {} does exist in {}."
                              " Skipping...".format(fname, download_dir))
                        continue
                    urllib.request.urlretrieve(url, os.path.join(download_dir, fname))