Dataset Walker

test.py

from datacat import client_from_config_file

client = client_from_config_file()

walker = Walker(client)

def doWalk(site):
    dscount = 0
    path = "/LSST/mirror/BNL-test/test/ITL-CCD/ITL-113-10-360Khz-g2/read_noise/v0"
    for root, containers, datasets in walker.walk(path, site=site):
        #print root
        #print [c for c in containers]
        #print [ds for ds in datasets]
        dscount += len(datasets)
    
    print "{} datasets in {} with site = {}".format(dscount, path, site)

doWalk(None) # Uses default
doWalk("all")
doWalk("BNL")
doWalk("slac.lca.archive")

walker.py

from datacat.model import Container

class Walker:
    
    def __init__(self, client):
        self.client = client
    
    def walk(self, top, site=None, topdown=True, onerror=None):
        
        isdir = lambda node: isinstance(node, Container)
        
        # We may not have read permission for top, in which case we can't
        # get a list of the files the directory contains.  os.path.walk
        # always suppressed the exception then, rather than blow up for a
        # minor reason when (say) a thousand readable directories are still
        # left to visit.  That logic is copied here.
        try:
            # Note that listdir and error are globals in this module due
            # to earlier import-*.
            nodes = self.client.children(top, site=site)
        except Exception as err:
            if onerror is not None:
                onerror(err)
            return
        
        dirs, nondirs = [], []
        for node in nodes:
            if isdir(node):
                dirs.append(node)
            else:
                nondirs.append(node)
        
        if topdown:
            yield top, dirs, nondirs
        for node in dirs:
            for x in self.walk(node.path, site, topdown, onerror):
                yield x
        if not topdown:
            yield top, dirs, nondirs


class SearchWalker:
    
    def __init__(self, client):
        self.client = client
    
    def walk(self, top, site=None, query=None, topdown=True, onerror=None):
        isdir = lambda node: isinstance(node, Container)
        try:
            nodes = self.client.children(top, site=site)
            nondirs = self.client.search(top, site=site, query=query)
        except Exception as err:
            if onerror is not None:
                onerror(err)
            return
        
        dirs = []
        for node in nodes:
            if isdir(node):
                dirs.append(node)
        
        if topdown:
            yield top, dirs, nondirs
        for node in dirs:
            for x in self.walk(node.path, site, query, topdown, onerror):
                yield x
        if not topdown:
            yield top, dirs, nondirs