geldoc.py

import argparse
import requests
import os
import os.path
import yaml
import time
import logging
from BeautifulSoup import BeautifulSoup

BASE_PATH = "geldoc/images/" # save directry path
URL = "http://10.0.1.240" # geldoc URL
USERNAME = 'xxxx'
PASSWORD = 'xxxx'

logging.basicConfig(level=logging.DEBUG, format="%(asctime)s|%(levelname)s|%(message)s")

def get_image(path):
    _url = "%s/%s" % (URL, path)
    req = requests.post(_url, auth=requests.auth.HTTPDigestAuth(USERNAME, PASSWORD))
    if req.ok:
        return req.content # image binary 
    else:
        return False
    
def save_image(image, outf):
    with open(os.path.join(BASE_PATH, outf), "wb") as f:
        f.write(image)
        
def get_all_gel_images():
    req = requests.post(URL, auth=requests.auth.HTTPDigestAuth(USERNAME, PASSWORD))
    if req.ok:
        html = req.content
        soup = BeautifulSoup(html)
        table = soup.findAll("a")
        for a in table:
            image_name = a.get("href").split("/")[0]
            if image_name.endswith('.jpeg') or image_name.endswith('.tiff'):
                raw_image = get_image(image_name)
                print "Downloading %s..." % (image_name)
                save_image(raw_image, image_name)

def _list_remote_gel_images():
    req = requests.post(URL, auth=requests.auth.HTTPDigestAuth(USERNAME, PASSWORD))
    if req.ok:
        data = {}
        html = req.content
        soup = BeautifulSoup(html)
        table = soup.findAll("a")
        for a in table:
            image_name = a.get("href").split("/")[0]
            if image_name.endswith('.jpeg') or image_name.endswith('.tiff'):
                data[str(image_name)] = 1
    return data

def _list_local_gel_images(path):
    data = {}
    for gel_f in filter(lambda x: x != ".DS_Store", os.listdir(path)):
        data[gel_f] = 1
    return data
    
def calculate_diff(_remote, _local):
    # load YAML
    remote_map = yaml.load(open(_remote).read())
    local_map = yaml.load(open(_local).read())
    
    # calculate diff between local and remote
    return set(remote_map.keys()) - set(local_map.keys())
    
def save_as_YAML(data, path):
    with open(path, "w") as f:
        yaml.dump(data, f)
        
def run(args):
    # Generate lists of the remote and local gel images
    logging.debug("Listing gel images from remote...")
    save_as_YAML(_list_remote_gel_images(), args.remote)
    logging.debug("Listing gel images in local...")
    save_as_YAML(_list_local_gel_images(BASE_PATH), args.local)
    
    # Downloaded newly generated images from remote
    new_images = calculate_diff(args.remote, args.local)
    if len(new_images) > 0:
        for image_path in new_images:
            logging.debug("Downloading new image <%s> from geldoc..." % (image_path))
            save_image(get_image(image_path), image_path)
            time.sleep(1)
    else:
        logging.debug("All geldoc images are already downloaded")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Automatically sync the 310's gel images")
    parser.add_argument("--remote", action="store", type=str, required=True, help='Path to remote YAML file')
    parser.add_argument("--local", action="store", type=str, required=True, help='Path to local YAML file')
    args = parser.parse_args()
    run(args)