Last active
May 27, 2020 10:24
-
-
Save tanmays/09210334644cf8e0516e5ed4f30bd8ed to your computer and use it in GitHub Desktop.
Python script to download files & folders recursively from a remote host via FTP
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
# Copyright (c) 2018, Tanmay Sonawane | |
# | |
# 3 clause/New BSD license: | |
# opensource: http://www.opensource.org/licenses/BSD-3-Clause | |
# wikipedia: http://en.wikipedia.org/wiki/BSD_licenses | |
# | |
#----------------------------------------------------------------------- | |
# This script allows to download all files and folders from a specific | |
# remote path to a local path recursively. | |
# | |
# I made this specifically to connect to my IP cam and download all video | |
# files to my NAS periodically. | |
# | |
# Use: | |
# You should run this as a cron job. An easy way is to just put this file | |
# in your NAS's /etc/cron.hourly/ folder. (remove the file extension) | |
# | |
# You can follow me on Twitter (@tanmays). | |
import os | |
import time | |
from sets import Set | |
from ftplib import FTP | |
########### CHANGE THESE ######################## | |
TITLE = 'Living Room Surveillance Videos' | |
SERVER = 'xxx.xxx.xxx.xxx' # your remote device's IP. It's best if you assign a static IP to the remote device. | |
USER = 'xxxx' # FTP username | |
PASS = 'xxxx' # FTP password | |
REMOTE_PATH = "/tmp/sd/record/" # path to copy files from, (ex. path is where the Yi Home 720p stores videos) | |
LOCAL_PATH = "/data/Videos/Surveillance/Living Room/" # local path, make sure the path is valid and folder exists (ex. path is where I store files on my NAS) | |
DELETE_AFTER_COPY = False # set true if you want to delete files from remote device after copying | |
NEW_FILES_ONLY = True # this will copy only new files, perfect for cron jobs | |
########################################### | |
def connectFTP(serverName, userName, passWord, remotePath, localPath, deleteRemoteFiles=False, onlyDiff=True): | |
try: | |
ftp = FTP(serverName) | |
except: | |
print "Couldn't find server" | |
ftp.login(userName,passWord) | |
ftp.cwd(remotePath) | |
try: | |
print "Connecting..." | |
copyFiles(ftp, remotePath, localPath, deleteRemoteFiles, onlyDiff) | |
except: | |
print "Connection Error - on: " + timeStamp() | |
print("Copying files from FTP complete. Closing FTP connection.") | |
ftp.close() # Close FTP connection | |
ftp = None | |
def copyFiles(ftp, remotePath, localPath, deleteRemoteFiles=False, onlyDiff=True): | |
ftp.cwd(remotePath) | |
lFileSet = Set(os.listdir(localPath)) | |
rFileSet = Set(ftp.nlst()) | |
transferListWithNewFiles = list(rFileSet - lFileSet) | |
transferList = ftp.nlst() | |
for fileOrFolder in transferList: | |
fullRemotePath = os.path.join(remotePath, fileOrFolder) | |
# Check for directories first | |
if is_FTP_Path_A_Directory(fullRemotePath, ftp): | |
# Check if directory exists at local | |
fullLocalPath = os.path.join(localPath, fileOrFolder) | |
if os.path.isdir(fullLocalPath): | |
# Directory exists in local too, copy files | |
copyFiles(ftp, fullRemotePath, fullLocalPath, deleteRemoteFiles, onlyDiff) | |
else: | |
# Create new directory | |
os.makedirs(fullLocalPath) | |
copyFiles(ftp, fullRemotePath, fullLocalPath, deleteRemoteFiles, onlyDiff) | |
else: | |
shouldCopy = False | |
if onlyDiff: | |
if fileOrFolder in transferListWithNewFiles: | |
shouldCopy = True | |
else: | |
shouldCopy = True | |
if shouldCopy == True: | |
# Create full local & remote filepaths | |
fullLocalFilePath = os.path.join(localPath, fileOrFolder) | |
fullRemoteFilePath = os.path.join(remotePath, fileOrFolder) | |
print("Copying file to local path: " + fullLocalFilePath) | |
# Open a file on local disk to write to | |
fileObj = open(fullLocalFilePath, 'wb') | |
# Download the file a chunk at a time using RETR | |
ftp.retrbinary('RETR ' + fullRemoteFilePath, fileObj.write) | |
# Close the file | |
fileObj.close() | |
# Delete the remote file if requested | |
if deleteRemoteFiles: | |
ftp.delete(fullRemoteFilePath) | |
def timeStamp(): | |
return str(time.strftime("%a %d %b %Y %I:%M:%S %p")) | |
def is_FTP_Path_A_Directory(path, ftp): | |
current = ftp.pwd() | |
try: | |
ftp.cwd(path) | |
except: | |
ftp.cwd(current) | |
return False | |
ftp.cwd(current) | |
return True | |
if __name__ == '__main__': | |
print ("\n---- Executing Script to copy "+ TITLE +" ----\n") | |
connectFTP(SERVER,USER,PASS,REMOTE_PATH,LOCAL_PATH,DELETE_AFTER_COPY,NEW_FILES_ONLY) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
great job
I would like to add a small function not to delete the file of the day. how can I proceed
thank you in advance