|
""" |
|
Automatically generated by Colaboratory. |
|
|
|
Original file is located at |
|
https://colab.research.google.com/drive/1P3o0_YObRlHx5z3m0h0MLxO9bxBaGaeO |
|
|
|
Dont forget to run the following command to install the required packages: |
|
pip install six python-firebase |
|
""" |
|
|
|
# Fill the server information |
|
ftp_server = 'ftp.cs.brown.edu' |
|
ftp_user = '' |
|
ftp_pass = '' |
|
|
|
# The ftp upload folder |
|
upload_folder = '/incoming' |
|
# The local path to extract to |
|
path = 'new' |
|
|
|
# Wilddog Authentication |
|
secret = '' |
|
email = '' |
|
|
|
# runs for ever? |
|
forever = False # False: run just once, True: runs forever and check the updates ever 10 minutes |
|
run_every = 10 * 60 # 10 minutes |
|
|
|
# Delete the record from wilddog after download |
|
delete_after_download = False # False: to keep, True: to delete |
|
|
|
import os |
|
import zipfile |
|
from ftplib import FTP |
|
from time import sleep |
|
|
|
from firebase import firebase |
|
from six.moves import urllib |
|
|
|
|
|
def enter_dir(f, path): |
|
try: |
|
f.cwd(path) |
|
except: |
|
try: |
|
f.mkd(path) |
|
f.cwd(path) |
|
except: |
|
pass |
|
print (path) |
|
|
|
|
|
if __name__ == '__main__': |
|
# connect |
|
authentication = None |
|
if email != '' and secret != '': |
|
authentication = authentication = firebase.FirebaseAuthentication(secret, email) |
|
wilddog = firebase.FirebaseApplication('https://site-eumt.wilddogio.com', authentication=authentication) |
|
|
|
# checking for updates and save to last_customs.txt |
|
# if there is an update check the listings |
|
last_customs = '' |
|
old_files = [] |
|
try: |
|
with open('last_customs.txt', 'r') as file: |
|
last_customs = file.readline() |
|
except: |
|
pass |
|
|
|
while True: |
|
|
|
result = wilddog.get('/listings_xml_aux/aux_xml_send_semaphore_customs', None) |
|
|
|
with open('last_customs.txt', 'w') as file: |
|
file.write(result) |
|
|
|
if last_customs != result: |
|
last_customs = result |
|
result = wilddog.get('/listings_xml_customs_id', None) |
|
print(result) |
|
|
|
try: |
|
# make sure that the local path exist |
|
os.makedirs(path) |
|
except OSError: |
|
pass |
|
# keep track of the old files |
|
old_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] |
|
|
|
# download the zip files and extract to path and delete the zip and listings_xml_customs_id from wilddog |
|
for key in result: |
|
value = result[key] |
|
print("Downloading:", value) |
|
zip_file = urllib.request.urlretrieve(value, os.path.join(path, value.split('/')[-1]))[0] |
|
try: |
|
with zipfile.ZipFile(zip_file, "r") as zip_ref: |
|
zip_ref.extractall(path) |
|
os.remove(zip_file) |
|
if delete_after_download: |
|
wilddog.delete('/listings_xml_customs_id', key) |
|
except Exception as e: |
|
print (e) |
|
|
|
# upload only the new files to the ftp server |
|
aftp = FTP(ftp_server, ftp_user, ftp_pass) |
|
aftp.login() |
|
enter_dir(aftp, upload_folder) |
|
|
|
all_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] |
|
new_files = set(all_files) - set(old_files) |
|
for file_name in new_files: |
|
with open(os.path.join(path, file_name), 'rb') as file: |
|
print("uploading:", file_name) |
|
aftp.storbinary('STOR %s' % file_name, file) |
|
|
|
aftp.quit() |
|
else: |
|
print('No updates') |
|
if forever is False: |
|
print ('DONE') |
|
break |
|
print('Sleeping for', run_every) |
|
sleep(run_every) |
