stk-terrain.py

# !/user/bin/env/python
# -*- coding: utf-8 -*-
# version: v0.0.1
# author: youth
# contact: [email protected]
# project: garbage
# filename: terrain.py.py
# datetime: 2017-08-31 16:05
# description:
import os
import math
import time
import json
import random
import redis
import requests
from multiprocessing.dummy import Pool
from fake_useragent import UserAgent

client = redis.StrictRedis(host='192.168.0.3', db=3)

ROOT_PATH = '/mnt/dem/tilesets/terrain'
ua = UserAgent()


# ROOT_PATH = '/Users/youngtwain/map/terrain'

def task(i):
    while True:
        url = client.rpop('dem')
        if url:
            try:
                time.sleep(random.choice(xrange(20)) / 10)
                headers = {
                    "Accept": "application/vnd.quantized-mesh;extensions=octvertexnormals-watermask,application/octet-stream;q=0.9,*/*;q=0.01",
                    "Accept-Encoding": "gzip, deflate",
                    'User-Agent': ua.random
                    # 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
                }
                url = 'https://assets.agi.com/stk-terrain/v1/tilesets/world/tiles/' + url
                url_list = url.split('/')
                path = os.path.join(ROOT_PATH, '/'.join((url_list[-3], url_list[-2])))
                filepath = os.path.join(ROOT_PATH, '/'.join((url_list[-3], url_list[-2], url_list[-1])))
                if os.path.exists(filepath):
                    continue
                r = requests.get(url, headers)
                if r.status_code == requests.codes.ok:

                    try:
                        if not os.path.exists(path):
                            os.makedirs(path)
                    except:
                        pass
                    f = open(filepath, 'wb')
                    for chunk in r.iter_content(chunk_size=512 * 1024):
                        if chunk:
                            f.write(chunk)
                    f.close()
                    print url, r.headers['content-length'], os.stat(filepath).st_size
            except:
                time.sleep(5)
                print 'restart crawl'
                client.lpush('dem', url)
        else:
            break


def producer():
    with open('layer.json', 'r') as f:
        data = json.loads(f.read())
        for idx, z in enumerate(data['available']):
            for i in z:
                for j in xrange(i['startX'], i['endX']+1):
                    for k in xrange(i['startY'], i['endY']+1):
                        url = '%s/%s/%s.terrain' % (str(idx), str(j), str(k))
                        client.rpush('dem', url)


                
def main():
    pool = Pool(50)
    pool.map(task, range(50))
    pool.close()
    pool.join()


if __name__ == '__main__':
    producer()
    main()