crawl_momo.py

# for envirment
import os
# for data
import pandas as pd
# for web
import requests
from lxml import html
from bs4 import BeautifulSoup as bs
import time

def get_api_data() :
    unix_time_str = str(round(time.time(),3)).replace('.','')
    header ={
        'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
        , 'authority' : 'www.momoshop.com.tw'
        , 'method' : 'POST'
        , 'path' : '/ajax/ajaxTool.jsp?n=2017' + unix_time_str
        , 'scheme' : 'https'
        , 'accept' : 'application/json, text/javascript, */*; q=0.01'
        , 'accept-encoding' : 'gzip, deflate, br'
        , 'accept-language' : 'en-GB,en-US;q=0.9,en;q=0.8,zh-TW;q=0.7,zh;q=0.6'
        , 'content-length' : '1000'
        , 'content-type' : 'application/x-www-form-urlencoded; charset=UTF-8'
        , 'cookie' : '_tam=F_I7I-3AcQceHa_4jxLxvjyw; __BWfp=c1589700520067x691f330a2; _gcl_marco=1.812941564.1591507116; customer=1; ck_mlu="MDkxMDkwNjQwMg=="; __auc=d86c709d17378dd051a91a8e2da; _ga=GA1.3.1342127572.1595460618; _mwa_uniCampaignInfo=1595460619931616265.1595460619931; _atrk_siteuid=iw90LvwuL0iEPQXH; _atrk_uid=b5de7fa1dbf9c3a9793116969b905aec78d3157d8bdbeb8bc4c51a8cf43fabbb; firstTimeOpenShop=forever; _gac_UA-22652017-1=1.1601521992.CjwKCAjw2dD7BRASEiwAWCtCb9ndk28UO194FxmLRMn9PfoxKatzq0BWWyIuMhSVH9q9EWigih9h-xoCt54QAvD_BwE; ARK_ID=JS3a11d73c5871a053c9117471bdd207a03a11; GoodsBrowsingHistory=7334385_1605263454/8175311_1602875845/8039425_1598996363/8039422_1598979903; JSESSIONID=59313BB57B5D5F3AA51B6E1CAD416FB6-m1.b1-shop35; wshop=wshop_web_b_35; bid=62a1a4268998bf7e52807db73bbff26e; isBI=1; __asc=42632873176135c7df152d10040; appier_utmz=%7B%7D; _atrk_ssid=W7ENYcS4_ehBfYyTpgL8yj; appier_page_isView_ERlDyPL9yO7gfOb=c1ba81a2b43eab37ca6849fd26ded6304b0b86714c6d2d837124d6226f2b1a19; _mwa_uniVisitorInfo=1595460619923102028.1595460619923.36.1606642597744; _gid=GA1.3.2066486355.1606642600; TN=undefined; CN=undefined; CM=undefined; _atrk_sessidx=6; appier_pv_counterERlDyPL9yO7gfOb=2; _mwa_uniSessionInfo=1606642597739936596.1606642597739.6.1606642842296'
        , 'dnt' : '1'
        , 'origin' : 'https://www.momoshop.com.tw'
        , 'referer' : 'https://www.momoshop.com.tw/category/DgrpCategory.jsp?d_code=1203000297'
        , 'sec-fetch-dest' : 'empty'
        , 'sec-fetch-mode' : 'cors'
        , 'sec-fetch-site' : 'same-origin'
        , 'x-requested-with' : 'XMLHttpRequest'
		}
    API_URL = {
        'API_URL': 'https://www.momoshop.com.tw/ajax/ajaxTool.jsp?n=2017&t=' + unix_time_str
		}
    post_data = {
        'data': '{"flag":2035,"data":{"params":{"keyword":"","checkedBrands":[],"checkedBrandsNo":[],"checkedAttrs":{},"checkedAttrsNo":{},"isBrandZone":false,"specialGoodsType":"","cateCode":"1203000297","cateLevel":"3","cp":"N","NAM":"N","normal":"N","first":"N","freeze":"N","superstore":"N","tvshop":"N","china":"N","tomorrow":"N","stockYN":"N","prefere":"N","threeHours":"N","curPage":"1","priceS":"","priceE":"","brandName":[],"sortType":"6","d_code":"1203000297"}}}'
    }
    session = requests.session()
    Post = session.post( API_URL['API_URL']
                                    , data = post_data
                                    , headers = header
                                    , timeout = 3
                                 )
    return Post

x = get_api_data()
print(x.text)

    # url_1 = 'https://www.momoshop.com.tw/category/DgrpCategory.jsp?d_code=1203000297'
    # url_2 = 'https://www.momoshop.com.tw/ecm/js/jquery-1.12.4.js?t=202011190001'
    # session.get(url_1, headers = header)
    # session.get(url_2, headers = header)

# 1606643286.3339832
# 1606643225306
# 1606643202
# 1606643382067

print('done!')