Last active
September 10, 2024 21:13
-
-
Save lucanello/1fa285804dff2eeb681d3a86f64cc03b to your computer and use it in GitHub Desktop.
Python Rclone Script with Telegram and InfluxDB Reporting
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Python Rclone Script with Telegram and InfluxDB Reporting | |
| # - This Script backups your data to any rclone location (see rclone.org) | |
| # - You can execute scripts, commands and database dumps before running a single rclone command | |
| # - It sends automatic error reports and summaries | |
| # - It saves all statistics and operations to an Influx database | |
| # - The data can be used for visualizations in e.g. Grafana | |
| # - You can automate it by using cron | |
| # Created by: Luca Koroll - https://github.com/lucanello | |
| # Last update: 2021-07-04 | |
| import json | |
| import os | |
| import requests | |
| import datetime | |
| import math | |
| import subprocess | |
| from influxdb import InfluxDBClient | |
| # Set to True if you want to make a test run | |
| dryrun = False | |
| # Specify logfile directory (where temporary logs are saved) | |
| logfiledir = '/root/' | |
| # Specify host tag for Influx data, used for filtering if you backup several hosts | |
| host = 'vhost' | |
| # Specify extra options, if you want to filter for example (see the calls below, where they are used) | |
| opts = ['--local-no-check-updated', '--delete-excluded', '--exclude-from=excluded.txt'] | |
| # Specify your Influx and Telegram credentials | |
| influxclient = InfluxDBClient('host', 8086, 'db_user', 'db_pass', 'db_name') | |
| telegram_token = "123456789:AAABBBCCCDDDEEEFFFGGGHHHIII" | |
| telegram_chat_id = "-123456789" | |
| result_msg = [] | |
| def rclone(source, target, configfile=None, transfers=48, dryrun=False, extra_opts=None, mode='sync', db_export=None, | |
| db_export_location=None, pre_cmd=None): | |
| # Construct logfilepath | |
| logfilepath = os.path.join(logfiledir + source.replace('/', '_') + '-rclone.log') | |
| # Construct rclone sync command | |
| basecmd = ('rclone ' + str(mode) + ' --transfers=' + str(transfers) + ' --log-file=' + str(logfilepath) + | |
| ' --use-json-log ' + '--log-level=INFO --stats=90m --fast-list').split() | |
| # Add dry-run for testing purposes | |
| basecmd.append('--dry-run') if dryrun else '' | |
| # Add explicit config if given | |
| basecmd.append('--config=' + str(configfile)) if configfile else '' | |
| # Add extra options | |
| if extra_opts: | |
| basecmd.extend(extra_opts) | |
| # Add source and target | |
| basecmd.append(str(source)) | |
| basecmd.append(str(target)) | |
| try: | |
| # Execute pre command | |
| if pre_cmd: | |
| result = subprocess.run(pre_cmd) | |
| if result.returncode != 0: | |
| raise Exception("Error in pre command: " + str(result)) | |
| # Execute database export | |
| if db_export: | |
| with open(db_export_location, "w") as outfile: | |
| result = subprocess.run(db_export, stdout=outfile) | |
| if result.returncode != 0: | |
| raise Exception("Error in database export: " + str(result)) | |
| # Execute command | |
| result = subprocess.run(basecmd) | |
| if result.returncode != 0: | |
| with open(logfilepath, 'r') as file: | |
| data = file.read() | |
| raise Exception(str(data) + " when executing: " + str(result)) | |
| # Parse logfile | |
| checks, transfers, bytes = parse_log(logfilepath, source, target) | |
| # Remove log | |
| os.remove(logfilepath) | |
| # Append to result message | |
| statistic = "\n ⤷ Checks: " + str(checks) + ", Transfers: " + str(transfers) + ", Bytes: " + \ | |
| str(convert_size(bytes)) | |
| icon = "✅ " if transfers > 0 else "🆗 " | |
| result_msg.append(icon + source + " -> " + target + statistic) | |
| except BaseException as e: | |
| notify_telegram(("<b>🚨 ERROR in Backup occurred 🚨</b>\n<i>Source</i>: %s\n<i>Target</i>: %s\n<i>Error</i>: %s" | |
| % (source, target, str(e)))) | |
| result_msg.append("❌ " + source + " -> " + target) | |
| def parse_log(logfile, source, target): | |
| with open(logfile) as f: | |
| data = [json.loads(line.rstrip()) for line in f if line[0] == "{"] | |
| stats = [] | |
| operations = [] | |
| for obj in data: | |
| if 'accounting/stats' in obj['source']: | |
| stats.append(obj) | |
| elif 'operations/operations' in obj['source']: | |
| operations.append(obj) | |
| for obj in stats: | |
| obj['stats']['speed'] = float(obj['stats']['speed']) | |
| json_body = [ | |
| { | |
| "measurement": "stats", | |
| "tags": { | |
| "host": host, | |
| "level": obj['level'], | |
| "log_entry_source": obj['source'], | |
| "source": source, | |
| "target": target | |
| }, | |
| "time": obj['time'], | |
| "fields": obj['stats'] | |
| } | |
| ] | |
| if not dryrun: | |
| influxclient.write_points(json_body) | |
| for obj in operations: | |
| json_body = [ | |
| { | |
| "measurement": "operations", | |
| "tags": { | |
| "host": host, | |
| "level": obj['level'], | |
| "log_entry_source": obj['source'], | |
| "objType": obj['objectType'], | |
| "msg": obj['msg'], | |
| "source": source, | |
| "target": target | |
| }, | |
| "time": obj['time'], | |
| "fields": { | |
| "obj": obj['object'] | |
| } | |
| } | |
| ] | |
| if not dryrun: | |
| influxclient.write_points(json_body) | |
| return stats[0]['stats']['totalChecks'], stats[0]['stats']['totalTransfers'], stats[0]['stats']['totalBytes'] | |
| def notify_telegram(text): | |
| params = {"parse_mode": "HTML", "chat_id": telegram_chat_id, "text": text} | |
| url = f"https://api.telegram.org/bot{telegram_token}/sendMessage" | |
| requests.post(url, params=params) | |
| def convert_size(size_bytes): | |
| if size_bytes == 0: | |
| return "0B" | |
| size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") | |
| i = int(math.floor(math.log(size_bytes, 1024))) | |
| p = math.pow(1024, i) | |
| s = round(size_bytes / p, 2) | |
| return "%s %s" % (s, size_name[i]) | |
| if __name__ == '__main__': | |
| start = datetime.datetime.now() | |
| # Using extra options (specified in the beginning) | |
| rclone('/source/folder', 'target:folder/', dryrun=dryrun, | |
| extra_opts=opts) | |
| # Add another extra option | |
| rclone('/source/folder', 'target:folder/', dryrun=dryrun, | |
| extra_opts=opts + ['--min-age=30m']) | |
| # Backup a database which is dumped in before | |
| # with rclone(..., db_expot=command, db_export_location=location) you can execute a dump to the given location | |
| db_export_cmd = '/usr/bin/mysqldump --no-tablespaces --skip-dump-date --single-transaction -h localhost -u db_user -pdb_pass db_name' | |
| rclone('/source/folder', 'target:folder/', | |
| dryrun=dryrun, extra_opts=opts, mode='copy', | |
| db_export=db_export_cmd.split(), db_export_location='/dump/target/db_export.sql') | |
| # Run a general command in before | |
| # I am using this for gitea backups for example | |
| pre_cmd = 'bash /sample/script.sh' | |
| rclone('/source/folder', 'target:folder/', | |
| dryrun=dryrun, pre_cmd=pre_cmd.split()) | |
| end = datetime.datetime.now() | |
| duration = end - start | |
| # Send Telegram notification | |
| notify_telegram(("<b>Backup Statistics:</b>\n\n%s\n\n<i>Start Time</i>: %s\n<i>End Time</i>: %s\n<i>Duration</i>: %s minutes" % | |
| ("\n\n".join(result_msg), start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S"), | |
| divmod(duration.total_seconds(), 60)[0]))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "__inputs": [ | |
| { | |
| "name": "DS_BACKUPS@INFLUXDB", | |
| "label": "backups@InfluxDB", | |
| "description": "", | |
| "type": "datasource", | |
| "pluginId": "influxdb", | |
| "pluginName": "InfluxDB" | |
| } | |
| ], | |
| "__requires": [ | |
| { | |
| "type": "panel", | |
| "id": "gauge", | |
| "name": "Gauge", | |
| "version": "" | |
| }, | |
| { | |
| "type": "grafana", | |
| "id": "grafana", | |
| "name": "Grafana", | |
| "version": "7.5.7" | |
| }, | |
| { | |
| "type": "datasource", | |
| "id": "influxdb", | |
| "name": "InfluxDB", | |
| "version": "1.0.0" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "piechart", | |
| "name": "Pie chart v2", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "stat", | |
| "name": "Stat", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "table", | |
| "name": "Table", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "timeseries", | |
| "name": "Time series", | |
| "version": "" | |
| } | |
| ], | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": "-- Grafana --", | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 0, | |
| "id": null, | |
| "links": [], | |
| "panels": [ | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "bars", | |
| "fillOpacity": 10, | |
| "gradientMode": "opacity", | |
| "hideFrom": { | |
| "graph": false, | |
| "legend": false, | |
| "tooltip": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "spanNulls": true | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| }, | |
| "unit": "decbytes" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 14, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "id": 10, | |
| "interval": "1d", | |
| "options": { | |
| "graph": {}, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltipOptions": { | |
| "mode": "single" | |
| } | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "$tag_source", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "source" | |
| ], | |
| "type": "tag" | |
| }, | |
| { | |
| "params": [ | |
| "null" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "totalBytes" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "mean" | |
| } | |
| ] | |
| ], | |
| "tags": [ | |
| { | |
| "key": "transfers", | |
| "operator": ">", | |
| "value": "0" | |
| } | |
| ] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Transferred data per source", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 5, | |
| "x": 14, | |
| "y": 0 | |
| }, | |
| "id": 8, | |
| "interval": "1d", | |
| "options": { | |
| "displayLabels": [], | |
| "legend": { | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "values": [] | |
| }, | |
| "pieType": "pie", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "text": {} | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "$tag_source", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "source" | |
| ], | |
| "type": "tag" | |
| } | |
| ], | |
| "limit": "", | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5", | |
| "rawQuery": false, | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "transfers" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "last" | |
| } | |
| ] | |
| ], | |
| "tags": [ | |
| { | |
| "key": "transfers", | |
| "operator": ">", | |
| "value": "0" | |
| } | |
| ] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Transfers by source", | |
| "type": "piechart" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [], | |
| "max": 5000000, | |
| "min": 0, | |
| "thresholds": { | |
| "mode": "percentage", | |
| "steps": [ | |
| { | |
| "color": "dark-red", | |
| "value": null | |
| }, | |
| { | |
| "color": "dark-yellow", | |
| "value": 50 | |
| }, | |
| { | |
| "color": "dark-green", | |
| "value": 75 | |
| } | |
| ] | |
| }, | |
| "unit": "Bps" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 5, | |
| "x": 19, | |
| "y": 0 | |
| }, | |
| "id": 12, | |
| "interval": "1d", | |
| "options": { | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showThresholdLabels": false, | |
| "showThresholdMarkers": true, | |
| "text": {} | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| } | |
| ], | |
| "limit": "", | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5", | |
| "rawQuery": false, | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "speed" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "mean" | |
| } | |
| ] | |
| ], | |
| "tags": [ | |
| { | |
| "key": "transfers", | |
| "operator": ">", | |
| "value": "0" | |
| } | |
| ] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Transfer speed", | |
| "type": "gauge" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "bars", | |
| "fillOpacity": 10, | |
| "gradientMode": "opacity", | |
| "hideFrom": { | |
| "graph": false, | |
| "legend": false, | |
| "tooltip": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "spanNulls": true | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| }, | |
| "unit": "none" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 14, | |
| "x": 0, | |
| "y": 8 | |
| }, | |
| "id": 2, | |
| "interval": "1d", | |
| "options": { | |
| "graph": {}, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltipOptions": { | |
| "mode": "single" | |
| } | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "$tag_msg", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "msg" | |
| ], | |
| "type": "tag" | |
| }, | |
| { | |
| "params": [ | |
| "0" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "measurement": "operations", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "obj" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "count" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Operation types", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 5, | |
| "x": 14, | |
| "y": 8 | |
| }, | |
| "id": 11, | |
| "interval": "1d", | |
| "options": { | |
| "displayLabels": [], | |
| "legend": { | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "values": [] | |
| }, | |
| "pieType": "pie", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "text": {} | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "$tag_msg", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "msg" | |
| ], | |
| "type": "tag" | |
| }, | |
| { | |
| "params": [ | |
| "0" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "measurement": "operations", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "obj" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "count" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Operation types", | |
| "type": "piechart" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "fixedColor": "dark-red", | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "dark-green", | |
| "value": null | |
| }, | |
| { | |
| "color": "dark-red", | |
| "value": 1 | |
| } | |
| ] | |
| }, | |
| "unit": "none" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 5, | |
| "x": 19, | |
| "y": 8 | |
| }, | |
| "id": 13, | |
| "interval": "1d", | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "text": {}, | |
| "textMode": "auto" | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| } | |
| ], | |
| "limit": "", | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5", | |
| "rawQuery": false, | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "errors" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "mean" | |
| } | |
| ] | |
| ], | |
| "tags": [ | |
| { | |
| "key": "transfers", | |
| "operator": ">", | |
| "value": "0" | |
| } | |
| ] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Errors", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "bars", | |
| "fillOpacity": 10, | |
| "gradientMode": "opacity", | |
| "hideFrom": { | |
| "graph": false, | |
| "legend": false, | |
| "tooltip": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "spanNulls": true | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| }, | |
| "unit": "decbytes" | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Checks" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "unit", | |
| "value": "none" | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Transfers" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "unit", | |
| "value": "none" | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 24, | |
| "x": 0, | |
| "y": 16 | |
| }, | |
| "id": 3, | |
| "interval": "1d", | |
| "options": { | |
| "graph": {}, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltipOptions": { | |
| "mode": "single" | |
| } | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "alias": "Bytes transferred", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "0" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "refId": "A", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "totalBytes" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "sum" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| }, | |
| { | |
| "alias": "Checks", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "0" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "hide": false, | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT sum(\"totalChecks\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", | |
| "rawQuery": false, | |
| "refId": "B", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "totalChecks" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "sum" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| }, | |
| { | |
| "alias": "Transfers", | |
| "groupBy": [ | |
| { | |
| "params": [ | |
| "$__interval" | |
| ], | |
| "type": "time" | |
| }, | |
| { | |
| "params": [ | |
| "0" | |
| ], | |
| "type": "fill" | |
| } | |
| ], | |
| "hide": false, | |
| "measurement": "stats", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT sum(\"totalChecks\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", | |
| "rawQuery": false, | |
| "refId": "C", | |
| "resultFormat": "time_series", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "totalTransfers" | |
| ], | |
| "type": "field" | |
| }, | |
| { | |
| "params": [], | |
| "type": "sum" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Operations", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${DS_BACKUPS@INFLUXDB}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "custom": { | |
| "align": null, | |
| "filterable": false | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Time" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 155 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "msg" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 92 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "source" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 462 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "msg" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "mappings", | |
| "value": [ | |
| { | |
| "from": "", | |
| "id": 1, | |
| "text": "Updated", | |
| "to": "", | |
| "type": 1, | |
| "value": "Updated modification time in destination" | |
| }, | |
| { | |
| "from": "", | |
| "id": 2, | |
| "text": "Updated", | |
| "to": "", | |
| "type": 1, | |
| "value": "Copied (replaced existing)" | |
| }, | |
| { | |
| "from": "", | |
| "id": 3, | |
| "text": "Added", | |
| "to": "", | |
| "type": 1, | |
| "value": "Copied (New)" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 12, | |
| "w": 24, | |
| "x": 0, | |
| "y": 26 | |
| }, | |
| "id": 5, | |
| "interval": "1d", | |
| "options": { | |
| "frameIndex": 0, | |
| "showHeader": true, | |
| "sortBy": [ | |
| { | |
| "desc": true, | |
| "displayName": "Time" | |
| } | |
| ] | |
| }, | |
| "pluginVersion": "7.5.7", | |
| "targets": [ | |
| { | |
| "groupBy": [], | |
| "measurement": "operations", | |
| "orderByTime": "ASC", | |
| "policy": "default", | |
| "query": "SELECT \"obj\" FROM \"operations\" WHERE (\"msg\" = 'Deleted') AND $timeFilter", | |
| "rawQuery": false, | |
| "refId": "A", | |
| "resultFormat": "table", | |
| "select": [ | |
| [ | |
| { | |
| "params": [ | |
| "msg" | |
| ], | |
| "type": "field" | |
| } | |
| ], | |
| [ | |
| { | |
| "params": [ | |
| "source" | |
| ], | |
| "type": "field" | |
| } | |
| ], | |
| [ | |
| { | |
| "params": [ | |
| "obj" | |
| ], | |
| "type": "field" | |
| } | |
| ] | |
| ], | |
| "tags": [] | |
| } | |
| ], | |
| "title": "File update log", | |
| "type": "table" | |
| } | |
| ], | |
| "refresh": false, | |
| "schemaVersion": 27, | |
| "style": "dark", | |
| "tags": [], | |
| "templating": { | |
| "list": [] | |
| }, | |
| "time": { | |
| "from": "now-7d", | |
| "to": "now" | |
| }, | |
| "timepicker": {}, | |
| "timezone": "", | |
| "title": "Backup Dashboard (Rclone)", | |
| "uid": "6xQTJPk7k", | |
| "version": 25 | |
| } |
could you provide some instruction to deploy it ?
Things you're going to need:
- Rclone set up with backup targets
- InfluxDB for metrics storage
- Grafana for visualization
- (optionally) a Telegram account and the chat ID, you might have to look up how to get it
Instructions:
- Copy the script to your machine.
- Install the specified python modules using
pip install <module>, you can see them imported at the beginning of the script. - You might change the options (line 22 to 35) to match your preferences.
- Change lines 162 to 181 to match your sources and targets you want to backup. You can use the sample commands. Just play around.
- Execute the script using
python3 rclone_script.py. Check the log to see if it's all running fine. - Check if the data is added to the Influx database.
- Log in to Grafana, add your InfluxDB as data source, create a new dashboard by importing the rclone_grafana_dashboard.json file and adapt it to the specified data source.
- You might setup a cronjob to execute the script regularly.
Please check the comments inside the script.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
could you provide some instruction to deploy it ?