Make backup and rotate files.

rotater.py

#!/bin/env ptyhon
# -*- coding: utf-8 -*-
# ======================================================================================================================
# Rotater
#   This script make backup in tar.gz format and remove files in target folder older
#   than today and builds a report file in where this script file path is.
#   Set this script as daily cronjob at midnight.
#   Author: Sencer HAMARAT
#   E-Mail: [email protected]
# ======================================================================================================================
# Libraries
import datetime
import os
import sys
import tarfile

# Configuration
# Folders to watch.
#                ("/full/path/to/folder",       Create, Backup, Remove)
WATCH_FOLDERS = [
                 ("/media/iphone",              False,   True,   True),
]

REPORT = "long"     # Detail of the report file "short", "long"
ROTATE_COUNT = 10   # How many days after remove old Backup and Log files.


# Variable settings

MY_LOCATION = os.path.dirname(os.path.realpath(__file__))
BACKUP_FOLDER = os.path.join(MY_LOCATION, "backup")         # Location is where this file exists
TODAY = datetime.date.today()
YESTERDAY = TODAY - datetime.timedelta(days=1)
ROTATE_DATE = TODAY - datetime.timedelta(days=ROTATE_COUNT)
LOG_FILE = os.path.join(BACKUP_FOLDER, "rotate_%s.log" % YESTERDAY.strftime("%Y-%m-%d"))


class Rotate(object):
    def __init__(self):
        self.folder, self.create, self.backup, self.clean = "", False, False, False
        self.backup_filename = ""
        self.file_list = []
        self.file_count = 0
        self.removed_files = {}
        self.removed_cnt = 0
        self.total_removed_cnt = 0
        self.err_on_remove = {}
        self.err_count = 0

    @staticmethod
    def is_folder_exists(folder):
        return os.path.isdir(folder)

    def check_folder(self, folder, make_if_not_exists=False):
        """
        Check existence of a folder. If it's not exists, create one regarding to parameter.
        :param folder: Full path to folder to be check
        :param make_if_not_exists: Make folder if not exists
        :return:
        """
        folder_exists = self.is_folder_exists(folder)
        if not folder_exists:
            write_to_file(
                "\n'%s' does not exists and " % folder
            )
            if make_if_not_exists:
                os.makedirs(folder)
                folder_exists = self.is_folder_exists(folder)
                write_to_file("created one.")
            else:
                write_to_file("creating this folder is not permitted in configuration.")
        return folder_exists

    def get_file_list(self):
        """
        Read file list from target folder in WATCH_FOLDERS list and apply date filter to file list.
        :return:
        """
        file_list_temp = os.listdir(self.folder)
        self.file_list = []
        for fl in file_list_temp:
            full_path = os.path.join(self.folder, fl)
            file_date = datetime.date.fromtimestamp(os.stat(full_path).st_mtime)
            if os.path.isfile(full_path) and not fl.startswith('.') and file_date < TODAY:
                self.file_list.append(fl)
        del file_list_temp

    def count_files(self):
        """
        File count in filtered self.file_list
        :return:
        """
        self.file_count = len(self.file_list)
        self._repot_statistics()

    def make_backup(self):
        """
        Make compressed backup file of the target folder in WATCH_FOLDERS list at BACKUP_FOLDER before doing anything.
        Depends on BACKUP setting of the target folder.
        :return:
        """
        if self.check_folder(BACKUP_FOLDER, make_if_not_exists=True):
            arch_name = "%s_folder_bckp_%s" % (self.folder.split("/")[-1], YESTERDAY.strftime("%Y-%m-%d"))
            self.backup_filename = "%s/%s.tar.gz" % (BACKUP_FOLDER, arch_name)
            tar = tarfile.open(self.backup_filename, "w:gz")
            for fl in self.file_list:
                tar.add(os.path.join(self.folder, fl))
            tar.close()
        self._report_backup()

    def remove_files(self):
        """
        Remove files under target folder in WATCH_FOLDERS list with matching files in self.file_list
        :return:
        """
        rmvd_fl = list()
        self.removed_cnt = 0
        self.err_count = 0
        for fl in self.file_list:
            full_path = os.path.join(self.folder, fl)
            file_date = datetime.date.fromtimestamp(os.stat(full_path).st_mtime)
            try:
                os.remove(full_path)
                rmvd_fl.append(full_path)
                self.removed_files.update({file_date: rmvd_fl})
                self.removed_cnt += 1
            except Exception as ure:
                self.err_on_remove.update({full_path: ure})
                self.err_count += 1
        self.total_removed_cnt += self.removed_cnt
        self._report_removed()

    def _report_error(self):
        """
        Append errors to self.report_tmp if any error exists.
        :return:
        """
        if bool(self.err_count):
            report_tmp = "\n\nError Count: ", self.err_count
            report_tmp += "\nErrors: "
            for key, value in self.err_on_remove.iteritems():
                report_tmp += "\n%s\n%s\n" % (key, value)
        else:
            report_tmp = "\n\nNo error reported..."
        write_to_file(report_tmp)

    def _repot_statistics(self):
        """
        Append total file count statistic to report
        :return:
        """
        write_to_file("\n%s\nTotal File Count: %s" % (self.folder, self.file_count))

    def _report_removed(self):
        """
        Append removed file count statistic to report
        :return:
        """
        write_to_file("\nRemoved File Count: %s" % self.removed_cnt)

    def _report_backup(self):
        """
        Append backup report
        :return:
        """
        if self.backup and bool(self.file_count):
            report_tmp = "\n\nRemoved files are backed up as %s" % self.backup_filename
        else:
            report_tmp = "\n\nNo backup made. Reasons: "
            if not self.backup:
                report_tmp += "Backup is disabled "
            if not bool(self.file_count):
                report_tmp += "No file available to backup "
        write_to_file(report_tmp)

    def _long_report(self):
        """
        With long report option, each processed file included into report.
        This will make report longer.
        :return:
        """
        report_tmp = ""
        if REPORT == "long" or bool(self.err_count):
            if bool(self.removed_files):
                report_tmp = "\nRemoved Files:"
                for key, value in self.removed_files.items():
                    if bool(value):
                        report_tmp += "\n    DATE: %s" % (key.strftime("%Y-%m-%d"))
                        for x in value:
                            report_tmp += "\n        %s" % x
            write_to_file(report_tmp)
            self._report_error()

    def rotate(self):
        """
        Self rotate files under BACKUP_FOLDER depending to ROTATE_DATE
        :return:
        """
        rotating_files = os.listdir(BACKUP_FOLDER)
        for fl in rotating_files:
            full_path = os.path.join(BACKUP_FOLDER, fl)
            fd_check = datetime.date.fromtimestamp(os.stat(full_path).st_mtime) < ROTATE_DATE
            fs_check = fl.startswith("%s_" % self.folder.split("/")[-1])
            fe_check = fl.endswith((".log", ".tar.gz"))
            if os.path.isfile(full_path) and fd_check and fs_check and fe_check:
                os.remove(full_path)

    def set_vars(self, folder):
        """
        Setting variables and formatting folder path string.
        :param folder:
        :return:
        """
        self.folder, self.create, self.backup, self.clean = folder
        self.folder = self.folder.rstrip("/")

    def run(self):
        """
        Make call to methods in order.
        :return:
        """
        write_to_file("File Statistics:\n\n")
        for folder in WATCH_FOLDERS:
            self.set_vars(folder)
            if self.check_folder(self.folder, make_if_not_exists=self.create):
                self.get_file_list()
                self.count_files()
                if self.backup:
                    self.make_backup()
                if self.clean:
                    self.remove_files()
                self._long_report()
            # Move rotate method to head if you want to create disk space before application start to creating files.
            self.rotate()
            write_to_file("\n")
        write_to_file("\nTotal removed files count: %s" % self.total_removed_cnt)


def write_to_file(text):
    """
    Write given text to file.
    :param text:
    :return:
    """
    with open(LOG_FILE, "a") as log_file:
        log_file.write(text)

if __name__ == "__main__":
    try:
        sc = Rotate()
        sc.run()
    except Exception as uee:
        write_to_file(repr(uee))
    finally:
        sys.exit()