generate-pinlist.py

generate-pinlist.py

#!/usr/bin/env python
# (c) 2017 Jeffrey Paul <[email protected]> - All rights reserved.

#import ipfsapi
from sanelogging import log
from datetime import datetime
import json
import os
import os.path
import re
import subprocess
import yaml

ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')

#def connectIpfs(hostname='127.0.0.1',port=5001):
#    ipfs = ipfsapi.connect(hostname,port)
#    log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
#    return ipfs

def main():
    startTime = datetime.utcnow()
    log.info("generating pin list.")
    generatePinList()
    elapsed = datetime.utcnow() - startTime
    log.info("finished in %is" % elapsed.total_seconds())

def generatePinList():
    pinFiles = readPinFiles()
#    ipfs = connectIpfs()

    resources = []

    for pinFile in pinFiles:
        rs = extractResources(pinFile)
        resources.extend(rs)

    log.info("collected data from files.")

    resources = resolveResources(resources)

    output = {
        "generatedAt": datetime.utcnow().strftime("%s"),
        "pinList": resources
    }
    log.info("resources resolved, writing pinlist file")
    outputFileName = 'pinlist.json'
    fd = open(outputFileName + '.new','w')
    json.dump(output,fd)
    fd.close()
    os.rename(outputFileName + '.new', outputFileName)

def extractResources(pinFile):
    # this pulls out the list of ipns/ipfs
    # items from the single file
    output = []
    if 'ipns' in pinFile['pins']:
        if isinstance(pinFile['pins']['ipns'], basestring):
            # single ipns name
            output.append({
                'type': 'ipns',
                'name': pinFile['pins']['ipns'],
                'fileName': os.path.basename(pinFile['fileName'])
            })
        else:
            # list of ipns names
            for ipnsName in pinFile['pins']['ipns']:
                output.append({
                    'type': 'ipns',
                    'name': ipnsName,
                    'fileName': os.path.basename(pinFile['fileName'])
                })

    if 'ipfs' in pinFile['pins']:
        if isinstance(pinFile['pins']['ipfs'], basestring):
            # single ipfs name
            output.append({
                'type': 'ipfs',
                'name': pinFile['pins']['ipfs'],
                'fileName': os.path.basename(pinFile['fileName'])
            })
        else:
            # list of ipfs names
            for ipfsName in pinFile['pins']['ipfs']:
                output.append({
                    'type': 'ipfs',
                    'name': ipfsName,
                    'fileName': os.path.basename(pinFile['fileName'])
                })
    return output

def resolveResources(resources):
    log.info("resolving resource names for pin list")
    for resource in resources:
        if resource['type'] == 'ipfs':
            resource['resolved'] = resource['name']
        elif resource['type'] == 'ipns':
            # attepmt to resolve ipns name
            # FIXME try/catch
            res = None
            try:
                res = resolveIpnsName(resource['name'])
                if res:
                    resource['resolved'] = res
            except LookupError:
                log.info(
                    "unable to resolve ipns name '%s' from file '%s'"
                        % (resource['name'], resource['fileName'])
                )
                resource['error'] = 'unable to resolve name'
    return resources

def runCommand(cmd):
    p = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT
    )
    retval = p.wait()
    if retval != 0:
        raise LookupError()
    return p.stdout.read()

def resolveIpnsName(ipnsName):
    log.debug("resolving name '%s'" % ipnsName)
    res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
    ipfsNameMatch = ipfsPathPattern.match(res)
    if not ipfsNameMatch:
        log.info("unable to resolve ipns name '%s'" % ipnsName)
        return None
    ipfsName = ipfsNameMatch.group(1)
    log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
    return ipfsName

def readPinFiles():
    log.info("reading pinfiles...")
    output = []
    pinFiles = getPinFileNames()
    for pinFileName in pinFiles:
        pinFileData = parsePinFile(pinFileName)
        pinFileData['fileName'] = pinFileName
        output.append(pinFileData)
    return output

def parsePinFile(fn):
    x = {}
    f = open(fn,"r")
    x = yaml.load(f)
    f.close()
    validatePinFileData(x)
    return x

def validatePinFileData(pinFileStruct):
    # FIXME validate various file properties here
    pass

def getPinFileNames():
    thisDir = os.path.dirname(os.path.abspath(__file__))
    pinDir = os.path.join(thisDir, 'pins')
    pinFileNames = [
        os.path.join(pinDir,f) for f in os.listdir(pinDir)
            if os.path.isfile(os.path.join(pinDir, f))
    ]
    return pinFileNames

if __name__ == "__main__":
    main()