duplicity, idrive, backend

idrivebackend.py

# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
#
# Copyright 2015 aappddeevv <[email protected]>
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import sys
import os.path
import urllib
import tempfile
import re
import xml.etree.ElementTree as ET
import uuid
import shutil

import duplicity.backend
from duplicity import globals
from duplicity import log
from duplicity import tempdir

"""IDrive backend.
Uses idevsutil command line utility. Environment variables must be set
that locates the idevsutil program and provides it default authentication
information. The duplicity url id and password information is ignored
and cannot not be used.

Install this backend python file into the /usr/libxx/pythonxx/site-packages/duplicity/backends 
folder then use idrive:///your-idrive-folder-for-backups for the duplicity url.
The extra slash indicates an absolute location starting at your idrive root.
Do not specify any other duplicity url options. All duplicity backup files
will be deposited into your-idrive-folder-for-backups. I typically use
a duplicity backup set name like --name myidrivedaily to keep my different backup
locations separate. The local sigtar and manifest files would then be cached in 
~/.cache/duplicity/myidrivedaily and your actual backup files would be
located at the URL you specified to duplicity.

Make sure that the idrive folder your-idrive-folder-for-backups already
exists on idrive before running duplicity. You may need to use the web 
interface to create it first or you could use the idevsutil --create-dir
command. duplicity does not create the target url folder hierarchy for
you.

If you are already using encryption on idrive you could run:

duplicity --num-retries 3 --no-encryption --volsize 2048 --name=idrivedaily ~/yourfolder idrive:///duplicity

You can try --asynchronous-upload but I have had more errors when this mode
is used versus sequential uploads. Because idrive is in the cloud, you need
to find the right combination of volsize, num-retries and asynchronous-upload
that work for you.

The backend uses uuid named directories at the top level idrive folder
to load files before moving them to the final idrive location. If you
see any uuid folders at the top level idrive root and if duplicity has
completed running, you can safely delete them.

You need to set two environment variables before use:

IDRIVEID: Set to your idrive id, which is usually an email address e.g. [email protected]
IDEVSUTIL: Set to the idevsutil program and default authentication variables e.g. /path/idevsutil --password-file=blah --pvt-key=keyfile.txt"

There is extensive debug information available if you use duplicity's
verbosity mode e.g. -v 9.
"""
class IDriveBackend(duplicity.backend.Backend):

    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)
        # parsed_url will have leading slashes in it, 4 slashes typically.
        self.parsed_url = parsed_url
        self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
        log.Debug("parsed_url: {0}".format(parsed_url))

        # get the command to run
        self.cmd = os.environ.get("IDEVSUTIL")
        if self.cmd is None:
            raise BackendException(_("No IDEVSUTIL environment variable set. Should contain idevsutil and auth args."))
        log.Debug("idrive command base: %s" % (self.cmd))
        
        # get the idrive id, which is often an email address
        self.idriveid = os.environ.get("IDRIVEID")
        if self.idriveid is None:
            raise BackendException(_("No IDRIVEID environment variable set. Should contain idrive id"))
        log.Debug("idrive id: %s" % (self.idriveid))
        
        # get the server address
        try:
            serverrequest = self.cmd + " " + " --getServerAddress "  + self.idriveid
            _, serverresponse, _ = self.subprocess_popen(serverrequest)
            el = ET.fromstring(serverresponse)
            self.idriveserver = el.attrib["cmdUtilityServer"]
        except KeyError:
            raise BackendException(_("Unable to obtain server address for user {0}".format(self.idriveid)))
        log.Debug("idrive server: {0}".format(self.idriveserver))
        

    def _put(self, source_path, remote_filename):
        """Put a file. 
        :param source_path: Local file system path. Absolute path.
        :param remote_filename: Has the duplicity url path already included and is expressed in the backend filesystem.
        """
        self.put_file(source_path.name, os.path.join(urllib.unquote(self.parsed_url.path.rstrip('/')), remote_filename.lstrip('/')))

    def _get(self, remote_filename, local_path):
        """Get a file.
        Download to a temporary directory then move the file to the final destination.
        """
        local_path_dirname = os.path.dirname(local_path.name)
        remote_path = os.path.join(urllib.unquote(self.parsed_url.path.lstrip('/')), remote_filename).rstrip()

        log.Debug("_get: remote_filename={0}, local_path={1}, remote_path={2}, parsed_url.path={3}".format(remote_filename, local_path, remote_path, self.parsed_url.path))

        tmpdir = tempfile.mkdtemp()
        log.Debug("_get created temporary download folder: {}".format(tmpdir))
        
        flist = tempfile.NamedTemporaryFile()
        flist.write(remote_path)
        flist.seek(0)
        
        commandline = (self.cmd + " --files-from={0} {1}@{2}::home/ {3}").format(flist.name, self.idriveid,
                                                                                 self.idriveserver, tmpdir)
        log.Debug("get command: {0}".format(commandline))

        _, getresponse, _ = self.subprocess_popen(commandline)
        flist.close()
        log.Debug("_get response: {0}".format(getresponse))

        # move to the final location
        downloadedSrcPath = os.path.join(tmpdir, remote_path.lstrip('/').rstrip('/'))
        log.Debug("_get moving file {0} to final location: {1}".format(downloadedSrcPath, local_path.name))
        os.rename(downloadedSrcPath, local_path.name)
        
        shutil.rmtree(tmpdir)
        
    def _list(self):
        # use the uri path as the path on idrive to list
        remote_path = os.path.join(urllib.unquote(self.parsed_url.path.lstrip('/'))).rstrip()

        commandline = (self.cmd + " --auth-list {0}@{1}::home/{2}".format(self.idriveid, self.idriveserver, remote_path))
        _, l, _ = self.subprocess_popen(commandline)
        log.Debug("list response: {0}".format(l))

        # Look for our files in the returned strings but filter the lines first
        # get a list of lists from data lines returned by idevsutil --auth-list
        filtered = map((lambda line: re.split('\[|\]', line)) , [x for x in l.splitlines() if x.startswith("[")])
        # shrink columns that are all whitespace
        filtered = map((lambda line: map((lambda c: c.strip()), line)), filtered)
        # remove whitespace only columns
        filtered = map((lambda cols: filter((lambda c: c!=''), cols)), filtered)
        filtered = [x[-1] for x in filtered]
        
        return filtered

    def _delete(self, filename):
        remote_file_path = os.path.join(urllib.unquote(self.parsed_url.path.lstrip('/').rstrip('/')), filename.lstrip('/'))
        log.Debug("delete: {0} from remote file path {1}".format(filename, remote_file_path))
        
        # create a file-list file
        flist = tempfile.NamedTemporaryFile()
        flist.write(remote_file_path)
        flist.seek(0)
        
        delrequest = (self.cmd + " --delete-items --files-from={0} {1}@{2}::home/").format(flist.name, self.idriveid, self.idriveserver)
        log.Debug("delete: {0}".format(delrequest))
        _, delresponse, _ = self.subprocess_popen(delrequest)
        log.Debug("delete response: {0}".format(delresponse))
        
        flist.close()


    def create_dir(self, dirpath):
        """Create a directory on the remote server
        :param dirpath: Directory path on remote server. Should be absolute path from root and ignores duplicity url.
        """
        commandline = (self.cmd + " --create-dir={0} {1}@{2}::home/").format(dirpath, self.idriveid, self.idriveserver, local_path_dirname)
        log.Debug("create_dir command: {0}".format(commandline))

        _, response, _ = self.subprocess_popen(commandline)
        log.Debug("cerate_dir response: {0}".format(getresponse))

        flist.close()


    def dump_file(self, fname):
        """Dump a file to the log for debugging purpopses.
        """
        with open(fname, 'r') as fin:
            log.Debug(fin.read())

    def put_file(self, local_path, remote_path):
        """Put a file on the remote system using a temporary location as an interim structure.
        Essentially, the file is re-rooted for the load, then moved to the final destination.
        The temporary folder is removed. If the command is interrupted, trash could be left
        on the remote system. remote_path ignores any duplicity command line url.
        :param local_path: Absolute full local path to file.
        :param remote_path: Absolute full remote path to file.
        """
        log.Debug("put_file: local_path={}, remote_path={}".format(local_path, remote_path))
        u = str(uuid.uuid4())
        log.Debug("put_file creating temporary remote folder {0}".format(u))

        flist = tempfile.NamedTemporaryFile()
        flist.write(local_path)
        flist.seek(0)
        
        putrequest = (self.cmd + " --files-from={0} / {1}@{2}::home/{3}").format(flist.name, self.idriveid, self.idriveserver, u)
        log.Debug("put_file put command: {0}".format(putrequest))
        _, putresponse, _ = self.subprocess_popen(putrequest)
        flist.close()
        log.Debug("put_file put response: {0}".format(putresponse))

        moveitrequest = (self.cmd + " --rename --old-path={0} --new-path={1} {2}@{3}::home/").format(
            os.path.join(u, local_path.lstrip('/')), remote_path, self.idriveid, self.idriveserver)
        log.Debug("put_file rename command: {0}".format(moveitrequest))
        _, moveitresponse, _ = self.subprocess_popen(moveitrequest)
        log.Debug("put_file rename response: {0}".format(moveitresponse))

        flist = tempfile.NamedTemporaryFile()
        flist.write(u)
        flist.seek(0)
        removetmprequest = (self.cmd + " --files-from={0} --delete-items {1}@{2}::home/").format(flist.name, self.idriveid, self.idriveserver)
        log.Debug("put_file remove tmp folder command: {0}".format(removetmprequest))
        _, removeitresponse,_ = self.subprocess_popen(removetmprequest)
        log.Debug("put_file remove tmp folder response: {0}".format(removeitresponse))
        flist.close()

    def __del__(self):
        pass

duplicity.backend.register_backend("idrive", IDriveBackend)