backup script for cloud files (openstack swift) for backups with retention

backupcloudfiles.py

import subprocess
import tempfile
import select
import os
import sys
import copy
import datetime

src = sys.argv[1]
dst = sys.argv[1]+"-backup"

def loadmd5sqlite(db, container, prefix):
    with tempfile.NamedTemporaryFile() as f:
        env=os.environ.copy()
        if prefix:
            env['PREFIX'] = prefix
        lst = subprocess.Popen(
            ['./list.py', container],
            stdout=subprocess.PIPE,
            env=env
        )
        tsv = subprocess.Popen(
            ['sed', 's/  %s/\t%s\t/' % (prefix.replace('/', r'\/'), container)],
            stdin=lst.stdout,
            stdout=subprocess.PIPE
        )
        f.write('''CREATE TABLE IF NOT EXISTS `cmp` (md5 text, type text, filename text);
.separator "\t"
.import /dev/stdin cmp 
''')
        f.flush()
        load = subprocess.Popen(
            ['sqlite3', '-cmd', '.read '+f.name, db],
            stdin=tsv.stdout
        )
        load.wait()

if True:
    try: os.unlink('compare.sqlite3')
    except: pass
    loadmd5sqlite('compare.sqlite3', src, '')
    loadmd5sqlite('compare.sqlite3', dst, 'current/')

# bestanden die overschreven zouden worden/verwijdert zijn verplaatsen naar YYYY-MM-DD)
with tempfile.NamedTemporaryFile() as f:
    f.write('''.separator "\t"
SELECT `filename` FROM `cmp` GROUP BY `filename` HAVING COUNT(*) = 2 AND MIN(`md5`) != MAX(`md5`);
SELECT `filename` FROM `cmp` GROUP BY `md5`, `filename` HAVING COUNT(*) = 1 AND `type` = "%(dst)s";
''' % {'dst':dst})
    f.flush()
    sqlite3 = subprocess.Popen(["sqlite3", "compare.sqlite3", ".read "+f.name], stdout=subprocess.PIPE)
    env=os.environ.copy()
    env['SRC'] = dst+"/current/"
    env['DST'] = "%s/%s/" % (dst, datetime.date.today().isoformat())
    env['ACTION'] = "move"
    swiftcopy = subprocess.Popen(["xargs", "./swiftcopy.py"], env=env, stdin=sqlite3.stdout)
    swiftcopy.wait()

# bestanden die nieuw zijn kopieren naar current
with tempfile.NamedTemporaryFile() as f:
    f.write('''.separator "\t"
SELECT `filename` FROM `cmp` GROUP BY `md5`, `filename` HAVING COUNT(*) = 1 AND `type` = %(src)s;
''' % {'src':src})
    f.flush()
    sqlite3 = subprocess.Popen(["sqlite3", "compare.sqlite3", ".read "+f.name], stdout=subprocess.PIPE)
    env=os.environ.copy()
    env['SRC'] = src
    env['DST'] = dst + "/current/"
    env['ACTION'] = "copy"
    swiftcopy = subprocess.Popen(["xargs", "./swiftcopy.py"], env=env, stdin=sqlite3.stdout)
    swiftcopy.wait()

swiftcopy.py

import swiftclient.service
import sys
import os

def parsecontainer(container):
    s = container.split('/')
    return  s[0], "/".join(s[1:])

srccontainer, srcprefix = parsecontainer(os.environ['SRC'])
dstcontainer, dstprefix = parsecontainer(os.environ['DST'])
move = os.environ['ACTION'] == "move"

objects = []
for arg in sys.argv[1:]:
    oldname = srcprefix+arg
    newname = dstprefix+arg
    if move and srccontainer == dstcontainer and oldname == newname:
        continue
    objects.append(
        swiftclient.service.SwiftCopyObject(oldname, options={
            "destination": "/%s/%s" % (dstcontainer, newname),
        })
    )

with swiftclient.service.SwiftService() as swift:
    for i in swift.copy(srccontainer, objects):
        if i["action"] == "copy_object" and i["success"]:
            #print srccontainer, srcprefix, dstcontainer, dstprefix, i["object"]
            if move:
                for ii in swift.delete(container=srccontainer, objects=[i["object"]]):
                    pass
                    #if not ii["success"]:
                    #    print ii