Yandex_traffic_tile_download.py

import os, time, datetime, sys

##### Usage:
# python script.py A B C D E F
# A = X start
# B = X end
# C = Y start
# D = Y end
# E = traffic Dir parent path
# F = wgetList1.txt, i.e. wget List file name. This will keep on updating in each while loop
# python /home/ubuntu1/traffic/downloadTiles.py 303814 303824 196399 197339 /home/ubuntu1/traffic wgetList-X-303814-303824.txt

# The following errorLog will append all log info in our log.txt file
errorLog = "BatchX-X:%s-%s,TS:%s," % (sys.argv[1], sys.argv[2], datetime.datetime.now().strftime("%m%d%y%a%H%M"))

try:
    #try:
    open(os.path.join(sys.argv[5], "lockFiles", sys.argv[6]), 'w').close()
    errorLog = errorLog + "LockFileCreated," # For logging purpose

    start_time = time.time()

    #try:
    wgetListFile = open(os.path.join(sys.argv[5], "wgetList", sys.argv[6]), 'w')
    for X in range(int(sys.argv[1]), int(sys.argv[2])):
        for Y in range(int(sys.argv[3]), int(sys.argv[4])):
            # This double for loop will create all 94000 links for all tile to download from yandex server
            trafficUrl = "https://jgo.maps.yandex.net/1.1/tiles?trf&l=trj,trje&lang=tr_TR&z=17&scale=1&x=%s&y=%s&tm=%s" % (X, Y, int(time.time()))
            wgetListFile.write("%s\n" % (trafficUrl))
    wgetListFile.close()

    todayDate = time.strftime("%m%d%Y")
    currentHour = time.strftime("%H")

    try:
        # It's better to use, always, try and catch approach in python
        os.makedirs(os.path.join(sys.argv[5], "liveTrafficDataDownloadDir", todayDate, currentHour))
        errorLog = errorLog + "lTDDDcreated," # For logging purpose
    except OSError as e:
        if not os.path.exists(os.path.join(sys.argv[5], "liveTrafficDataDownloadDir", todayDate, currentHour)):
            errorLog = errorLog + "SERIOUSerrorInMkdirWithError " + str(e)
        else:
            errorLog = errorLog + "lTDDDalreadyThere," # For logging purpose

    os.chdir(os.path.join(sys.argv[5], "liveTrafficDataDownloadDir", todayDate, currentHour)) # Move to required dir
    os.system("wget -i %s" % os.path.join(sys.argv[5], "wgetList", sys.argv[6])) # Download all urls from wget file using one wget cmd.
    errorLog = errorLog + "wgetBatchDownload," # For logging purpose

    os.remove(os.path.join(sys.argv[5], "wgetList", sys.argv[6]))
    os.remove(os.path.join(sys.argv[5], "lockFiles", sys.argv[6]))
    errorLog = errorLog + "wgetList/lockFilesDeleted,"
    errorLog = errorLog + "timeTakenInSec:" + str(time.time() - start_time) # For logging purpose

except:
    errorLog = errorLog + "EXCEPThappened:%s" % (sys.exc_info()[0])


# The following try-except will log the execution time and other stuff of each wget file download for analysis purposes.
try:
    logFile = open(os.path.join(sys.argv[5], "log.txt"), 'a')
    logFile.write(errorLog)
    logFile.write("\n")
    logFile.close()
except IOError:
    logFile = open(os.path.join(sys.argv[5], "log.txt"), 'w')
    logFile.write(errorLog)
    logFile.write("\n")
    logFile.close()