Example of a high-level python API to VOSpace.

vospaceio.py

#!/usr/bin/env python
# encoding: utf-8
"""
File I/O using CANFAR's VOSpace.

2013-05-24 - Created by Jonathan Sick
"""

import os
import time
import logging
import subprocess

import vos  # https://github.com/ijiraq/cadcVOFS


class VOSpaceClient(object):
    """Interface to CANFAR/VOSpace.
    
    Handles downloading and uplooading files from VOSpace, including
    obtaining a .pem security certificate.
    """
    def __init__(self):
        super(VOSpaceClient, self).__init__()
        self._c = vos.Client()
        self.cert_path = os.path.expanduser("~/.ssl/cadcproxy.pem")
        self._days_valid = 7  # days a certificate is valid for
        if not os.path.exists(self.cert_path):
            self._get_certificate()

    def _get_certificate(self):
        """Download VOSpace certificate.
        
        We assume that the user's credentials are stored in a ~/.netrc file.
        
        machine www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login gmarx password swordfish
        machine www1.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login gmarx password swordfish
        machine www2.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login gmarx password swordfish
        machine www3.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login gmarx password swordfish
        machine www4.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login gmarx password swordfish

        See https://wiki.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/canfar/index.php/In_Depth_VOSpace
        """
        certDir = os.path.dirname(os.path.expanduser(self.cert_path))
        if not os.path.isdir(certDir):
            os.makedirs(certDir)
        if os.path.exists(self.cert_path):
            os.remove(self.cert_path)
        certURL = "http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/cred/proxyCert?daysValid=%i" % self._days_valid
        wgetCall = 'wget -O %s "%s"' % (self.cert_path, certURL)
        logging.debug("VOSpaceClient requesting new SSL certificate")
        subprocess.call(wgetCall, shell=True)
        print "Certfificate is readable",
        print os.access(self.cert_path, os.F_OK)
        # Client needs to be refreshed with new certificate
        self._c = vos.Client()

    def get(self, remoteName, remoteDir, localDir, user="jonathansick",
            maxtries=30, wait=60):
        """Copies file `remoteName` in `remoteDir` into `localDir`.
        
        remoteName : str
            Name of the file (filename and extension; excluding directory).
            A list of names can also be provided; assuming they are all in
            the same directory.
        remoteDir : str
            Directory on remote machine containing the files.
        localDir : str
            Directory where files will be downloaded into. Use `''` to download
            files into the current working directory.
        user : str
            Username on VOSpace.
        maxtries : int
            Maximum number of attempts to download the file. Default 30.
        wait : int
            Seconds to wait between retries. Default is 60 seconds.
        """
        localPath = os.path.join(localDir, remoteName)
        remotePath = "vos:" + os.path.join(user, remoteDir, remoteName)
        print "remotePath", remotePath
        # This pattern is suggested by
        # https://wiki.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/canfar/index.php/Python_script
        ntries = 1
        refreshedCert = False
        while not os.path.exists(localPath):
            try:
                self._c.copy(remotePath, localPath)
                logging.debug("VOSpaceClient.get() success for %s"
                        % remotePath)
            except IOError:
                if not refreshedCert:
                    self._get_certificate()
                    refreshedCert = True
                if not os.access(os.path.expanduser(self.cert_path), os.F_OK):
                    self._get_certificate()
            time.sleep(wait)
            logging.debug("VOSpaceClient.get() will retry for %s" % remotePath)
            ntries += 1
            if ntries > maxtries:
                raise VOSTransferTimeout

    def put(self, localPath, remoteDir, user="jonathansick", maxtries=120,
            wait=60):
        """Uploads file `localPath` to `remoteDir`.

        We use a modified exception pattern to the one suggested by
        https://wiki.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/canfar/index.php/Python_script
        In addition to verifying that the file size is correct, we also catch
        IOError events. A ten second wait between tries is instituted.
        
        localPath : str
            Full path to the local file that will be uploaded.
        remoteDir : str
            Name of the directory to upload the file into (excluding the
            filename).
        user : str
            Username on VOSpace.
        maxtries : int
            Maximum number of attempts to download the file. Default 120.
        wait : int
            Seconds to wait between retries. Default 60.
        """
        remotePath = "vos:" + os.path.join(user, remoteDir,
                os.path.basename(localPath))
        ntries = 0
        refreshedCert = False
        while 1:
            try:
                ntries += 1
                result = self._c.copy(localPath, remotePath)
                
                if result == os.stat(localPath).st_size:
                    logging.debug("VOSpaceClient.put() success for %s"
                            % remotePath)
                    return
                elif ntries > maxtries:
                    raise VOSTransferTimeout
                else:
                    logging.debug("VOSpaceClient.put() will rety for %s"
                            % remotePath)
                    time.sleep(wait)
            except IOError:
                if not refreshedCert:
                    self._get_certificate()
                    refreshedCert = True
                if ntries > maxtries:
                    raise VOSTransferTimeout
                if not os.access(os.path.expanduser(self.cert_path), os.F_OK):
                    self._get_certificate()
                logging.debug("VOSpaceClient.put() will rety for %s"
                        % remotePath)
                time.sleep(wait)
            if ntries > maxtries:
                raise VOSTransferTimeout

    def list(self, remote_dir, user="jonathansick"):
        """List file names in a directory."""
        remote_path = "vos:" + os.path.join(user, remote_dir)
        filenames = self._c.listdir(remote_path)
        return [os.path.join(remote_dir, name) for name in filenames]


class VOSTransferTimeout(Exception):
    pass