My weewx mods for sftp and for reducing the frequency of uploads.

ftpupload.py

#
#    Copyright (c) 2009-2020 Tom Keffer <[email protected]>
#
#    See the file LICENSE.txt for your full rights.
#
#    ** Hacked on by cranfill for sftp, July 2020 - based on weewx v. 4.1.1
#
"""For uploading files to a remove server via FTP or sftp"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import with_statement

import ftplib
import logging
import os
import sys
import time

from six.moves import cPickle

# cranfill: added pysftp (0.2.9 is the latest) from https://pypi.org/project/pysftp/
import pysftp
import datetime


log = logging.getLogger(__name__)


class FtpUpload(object):
    """Uploads a directory and all its descendants to a remote server.

    Keeps track of when a file was last uploaded, so it is uploaded only
    if its modification time is newer."""


    def __init__(self, server,
                 user, password,
                 local_root, remote_root,
                 port=21,
                 name="FTP",
                 passive=True,
                 secure=False,
                 debug=0,
                 secure_data=True,
                 reuse_ssl=False):
        """Initialize an instance of FtpUpload.

        After initializing, call method run() to perform the upload.

        server: The remote server to which the files are to be uploaded.

        user,
        password : The user name and password that are to be used.

        name: A unique name to be given for this FTP session. This allows more
        than one session to be uploading from the same local directory. [Optional.
        Default is 'FTP'.]

        passive: True to use passive mode; False to use active mode. [Optional.
        Default is True (passive mode)]

        secure: Set to True to attempt an FTP over TLS (FTPS) session.

        debug: Set to 1 for extra debug information, 0 otherwise.

        secure_data: If a secure session is requested (option secure=True),
        should we attempt a secure data connection as well? This option is useful
        due to a bug in the Python FTP client library. See Issue #284.
        [Optional. Default is True]

        reuse_ssl: Work around a bug in the Python library that closes ssl sockets that should
        be reused. See https://bit.ly/3dKq4JY [Optional. Default is False]
        """
        self.server = server
        self.user = user
        self.password = password
        self.local_root = os.path.normpath(local_root)
        self.remote_root = os.path.normpath(remote_root)
        self.port = port
        self.name = name
        self.passive = passive
        self.secure = secure
        self.debug = debug
        self.secure_data = secure_data
        self.reuse_ssl = reuse_ssl

# cranfill - new member var to indicate 'use actual sftp - not ftps'
        log.debug("ftpupload: **** creating a new ftpupload object")
        self.useSFTP = True

        if self.reuse_ssl and sys.version < '3.6':
            raise ValueError("Reusing an SSL connection requires Python version 3.6 or greater")

    def run(self):
        """Perform the actual upload.

        returns: the number of files uploaded."""

        n_uploaded = 0

# cranfill - only do upload every when minutes mod 5 is zero
        minute = datetime.datetime.now().minute
        log.debug(f"minute is {minute}")
        if minute % 5 == 0:
            log.debug("ftpupload: **** Proceeding to upload....")
        else:
            log.debug("ftpupload: **** Skipping upload")
            return n_uploaded # which is to say zero.

        # Get the timestamp and members of the last upload:
        timestamp, fileset = self.get_last_upload()

        try:
            # cranfill: SFTP? (Coders: From here on, search for 'self.useSFTP' to find cran's mods.)
            if self.useSFTP:
                # no equivalent operation for sftp
                pass
            else:
                if self.secure:
                    log.debug("Attempting secure connection to %s", self.server)
                    if self.reuse_ssl:
                        # Activate the workaround for the Python ftplib library.
                        from ssl import SSLSocket

                        class ReusedSslSocket(SSLSocket):
                            def unwrap(self):
                                pass

                        class WeeFTPTLS(ftplib.FTP_TLS):
                            """Explicit FTPS, with shared TLS session"""

                            def ntransfercmd(self, cmd, rest=None):
                                conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest)
                                if self._prot_p:
                                    conn = self.context.wrap_socket(conn,
                                                                    server_hostname=self.host,
                                                                    session=self.sock.session)
                                    conn.__class__ = ReusedSslSocket
                                return conn, size
                        log.debug("Reusing SSL connections.")
                        ftp_server = WeeFTPTLS()
                    else:
                        ftp_server = ftplib.FTP_TLS()
                else:
                    log.debug("Attempting connection to %s", self.server)
                    ftp_server = ftplib.FTP()

            if self.debug >= 2:
                ftp_server.set_debuglevel(self.debug)

            if self.useSFTP:
                log.debug("ftpupload: Attempting sftp connection to %s" % self.server)
                try:
                    sftp = pysftp.Connection(self.server, username=self.user, password=self.password)
                    # cnopts = sftp.CnOpts()
                    # cnopts.hostkeys = None
                except Exception as e:
                    log.debug("ftpupload: sftp error: %s" % e)
            else:
                ftp_server.set_pasv(self.passive)
                ftp_server.connect(self.server, self.port)
                ftp_server.login(self.user, self.password)
                if self.secure and self.secure_data:
                    ftp_server.prot_p()
                    log.debug("Secure data connection to %s", self.server)
                else:
                    log.debug("Connected to %s", self.server)

            # Walk the local directory structure
            for (dirpath, unused_dirnames, filenames) in os.walk(self.local_root):

                # Strip out the common local root directory. What is left
                # will be the relative directory both locally and remotely.
                local_rel_dir_path = dirpath.replace(self.local_root, '.')
                if _skip_this_dir(local_rel_dir_path):
                    continue
                # This is the absolute path to the remote directory:
                remote_dir_path = os.path.normpath(os.path.join(self.remote_root,
                                                                local_rel_dir_path))


                if self.useSFTP:
                    #
                    # FIXME This doesn't create the remote dir if it doesn't exist.
					# FIXME   As a matter of fact, this will FAIL if the server's directory structure
					# FIXME   doesn't match the source dir structure.
					#
                    log.debug("ftpupload: change remote dir to %s" % remote_dir_path)
                    sftp.chdir(remote_dir_path)
                else:
                    # Make the remote directory if necessary:
                    _make_remote_dir(ftp_server, remote_dir_path)

                # Now iterate over all members of the local directory:
                for filename in filenames:

                    full_local_path = os.path.join(dirpath, filename)
                    # See if this file can be skipped:
                    if _skip_this_file(timestamp, fileset, full_local_path):
                        continue

                    full_remote_path = os.path.join(remote_dir_path, filename)

                    if self.useSFTP:
                        log.debug("ftpupload: sftp: sending %s as %s" % (full_local_path, filename))
                        sftp.put(full_local_path, remotepath=full_remote_path)
                    else:
                        stor_cmd = "STOR %s" % full_remote_path
                        with open(full_local_path, 'rb') as fd:
                            try:
                                ftp_server.storbinary(stor_cmd, fd)
                            except ftplib.all_errors as e:
                                # Unsuccessful. Log it, then reraise the exception
                                log.error("Failed uploading %s to server %s. Reason: '%s'",
                                          full_local_path, self.server, e)
                                raise
                    # Success.
                    n_uploaded += 1
                    fileset.add(full_local_path)
                    log.debug("Uploaded file %s to %s", full_local_path, full_remote_path)
        finally:
            # cranfill: no need
            if self.useSFTP:
                pass
            else:
                try:
                    ftp_server.quit()
                except Exception:
                    pass

        timestamp = time.time()
        self.save_last_upload(timestamp, fileset)
        return n_uploaded

    def get_last_upload(self):
        """Reads the time and members of the last upload from the local root"""

        timestamp_file_path = os.path.join(self.local_root, "#%s.last" % self.name)

        # If the file does not exist, an IOError exception will be raised.
        # If the file exists, but is truncated, an EOFError will be raised.
        # Either way, be prepared to catch it.
        try:
            with open(timestamp_file_path, "rb") as f:
                timestamp = cPickle.load(f)
                fileset = cPickle.load(f)
        except (IOError, EOFError, cPickle.PickleError, AttributeError):
            timestamp = 0
            fileset = set()
            # Either the file does not exist, or it is garbled.
            # Either way, it's safe to remove it.
            try:
                os.remove(timestamp_file_path)
            except OSError:
                pass

        return timestamp, fileset

    def save_last_upload(self, timestamp, fileset):
        """Saves the time and members of the last upload in the local root."""
        timestamp_file_path = os.path.join(self.local_root, "#%s.last" % self.name)
        with open(timestamp_file_path, "wb") as f:
            cPickle.dump(timestamp, f)
            cPickle.dump(fileset, f)


def _skip_this_file(timestamp, fileset, full_local_path):
    """Determine whether to skip a specific file."""

    filename = os.path.basename(full_local_path)
    if filename[-1] == '~' or filename[0] == '#':
        return True

    if full_local_path not in fileset:
        return False

    if os.stat(full_local_path).st_mtime > timestamp:
        return False

    # Filename is in the set, and is up to date.
    return True


def _skip_this_dir(local_dir):
    """Determine whether to skip a directory."""

    return os.path.basename(local_dir) in ('.svn', 'CVS')


def _make_remote_dir(ftp_server, remote_dir_path):
    """Make a remote directory if necessary."""

    try:
        ftp_server.mkd(remote_dir_path)
    except ftplib.all_errors as e:
        # Got an exception. It might be because the remote directory already exists:
        if sys.exc_info()[0] is ftplib.error_perm:
            msg = str(e).strip()
            # If a directory already exists, some servers respond with a '550' ("Requested
            # action not taken") code, others with a '521' ("Access denied" or "Pathname
            # already exists") code.
            if msg.startswith('550') or msg.startswith('521'):
                # Directory already exists
                return
        # It's a real error. Re-raise the exception.
        raise

    log.debug("Made directory %s", remote_dir_path)