Skip to content

Instantly share code, notes, and snippets.

@tanmays
Last active May 27, 2020 10:24
Show Gist options
  • Save tanmays/09210334644cf8e0516e5ed4f30bd8ed to your computer and use it in GitHub Desktop.
Save tanmays/09210334644cf8e0516e5ed4f30bd8ed to your computer and use it in GitHub Desktop.
Python script to download files & folders recursively from a remote host via FTP
#!/usr/bin/python
# Copyright (c) 2018, Tanmay Sonawane
#
# 3 clause/New BSD license:
# opensource: http://www.opensource.org/licenses/BSD-3-Clause
# wikipedia: http://en.wikipedia.org/wiki/BSD_licenses
#
#-----------------------------------------------------------------------
# This script allows to download all files and folders from a specific
# remote path to a local path recursively.
#
# I made this specifically to connect to my IP cam and download all video
# files to my NAS periodically.
#
# Use:
# You should run this as a cron job. An easy way is to just put this file
# in your NAS's /etc/cron.hourly/ folder. (remove the file extension)
#
# You can follow me on Twitter (@tanmays).
import os
import time
from sets import Set
from ftplib import FTP
########### CHANGE THESE ########################
TITLE = 'Living Room Surveillance Videos'
SERVER = 'xxx.xxx.xxx.xxx' # your remote device's IP. It's best if you assign a static IP to the remote device.
USER = 'xxxx' # FTP username
PASS = 'xxxx' # FTP password
REMOTE_PATH = "/tmp/sd/record/" # path to copy files from, (ex. path is where the Yi Home 720p stores videos)
LOCAL_PATH = "/data/Videos/Surveillance/Living Room/" # local path, make sure the path is valid and folder exists (ex. path is where I store files on my NAS)
DELETE_AFTER_COPY = False # set true if you want to delete files from remote device after copying
NEW_FILES_ONLY = True # this will copy only new files, perfect for cron jobs
###########################################
def connectFTP(serverName, userName, passWord, remotePath, localPath, deleteRemoteFiles=False, onlyDiff=True):
try:
ftp = FTP(serverName)
except:
print "Couldn't find server"
ftp.login(userName,passWord)
ftp.cwd(remotePath)
try:
print "Connecting..."
copyFiles(ftp, remotePath, localPath, deleteRemoteFiles, onlyDiff)
except:
print "Connection Error - on: " + timeStamp()
print("Copying files from FTP complete. Closing FTP connection.")
ftp.close() # Close FTP connection
ftp = None
def copyFiles(ftp, remotePath, localPath, deleteRemoteFiles=False, onlyDiff=True):
ftp.cwd(remotePath)
lFileSet = Set(os.listdir(localPath))
rFileSet = Set(ftp.nlst())
transferListWithNewFiles = list(rFileSet - lFileSet)
transferList = ftp.nlst()
for fileOrFolder in transferList:
fullRemotePath = os.path.join(remotePath, fileOrFolder)
# Check for directories first
if is_FTP_Path_A_Directory(fullRemotePath, ftp):
# Check if directory exists at local
fullLocalPath = os.path.join(localPath, fileOrFolder)
if os.path.isdir(fullLocalPath):
# Directory exists in local too, copy files
copyFiles(ftp, fullRemotePath, fullLocalPath, deleteRemoteFiles, onlyDiff)
else:
# Create new directory
os.makedirs(fullLocalPath)
copyFiles(ftp, fullRemotePath, fullLocalPath, deleteRemoteFiles, onlyDiff)
else:
shouldCopy = False
if onlyDiff:
if fileOrFolder in transferListWithNewFiles:
shouldCopy = True
else:
shouldCopy = True
if shouldCopy == True:
# Create full local & remote filepaths
fullLocalFilePath = os.path.join(localPath, fileOrFolder)
fullRemoteFilePath = os.path.join(remotePath, fileOrFolder)
print("Copying file to local path: " + fullLocalFilePath)
# Open a file on local disk to write to
fileObj = open(fullLocalFilePath, 'wb')
# Download the file a chunk at a time using RETR
ftp.retrbinary('RETR ' + fullRemoteFilePath, fileObj.write)
# Close the file
fileObj.close()
# Delete the remote file if requested
if deleteRemoteFiles:
ftp.delete(fullRemoteFilePath)
def timeStamp():
return str(time.strftime("%a %d %b %Y %I:%M:%S %p"))
def is_FTP_Path_A_Directory(path, ftp):
current = ftp.pwd()
try:
ftp.cwd(path)
except:
ftp.cwd(current)
return False
ftp.cwd(current)
return True
if __name__ == '__main__':
print ("\n---- Executing Script to copy "+ TITLE +" ----\n")
connectFTP(SERVER,USER,PASS,REMOTE_PATH,LOCAL_PATH,DELETE_AFTER_COPY,NEW_FILES_ONLY)
@ccatarino
Copy link

great job
I would like to add a small function not to delete the file of the day. how can I proceed
thank you in advance

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment