cnbeta python script

cnbeta.py

# -*- utf-8 -*-

import os
import urllib.request
import urllib.parse
import sys
import json
import random
from getproxy import GetProxy
import time


#filename = 'proxy.json'

def getSId(params):
    for x,y in params:
        if x == "op":
            sid = y.split(',')[1]
            return sid
    
    return ''

def start(x, url, option, sid):
    
    '''
        #create the object, assign it to a variable
        proxy = urllib.request.ProxyHandler(x)
        # construct a new opener using your proxy settings
        opener = urllib.request.build_opener(proxy)
        # install the openen on the module-level
        urllib.request.install_opener(opener)
    ''' 
    print(x)
    
    cookie = '_awl=3.7.2-4ae5-39c67d431efa60736f041e2bca704957-58bf92bd-f-29ee2-372-1--5-0-; __utmz=208385984.1490676553.1.1.utmccn=(direct)|utmcsr=(direct)|utmcmd=(none); BDTUJIAID=6acc55f543824720e16d34f7dc86a7b5; __utma=208385984.2106185758.1490676553.1493954863.1496639101.13; Hm_lvt_4216c57ef1855492a9281acd553f8a6e=1496639102; _csrf=5a3ce99d4173c5d85b78510cf4af5bcf255103a3385582fa95180772bf77f859a%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22IZ48KDPBiI9852N3PmGfLYZdUfFDskYb%22%3B%7D; PHPSESSID=uabq6slbccd4fmugjlpu0j5sth';
    
        
    headers={
        'Host': 'www.cnbeta.com',
        'Cookie': cookie,
        'Accept':'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding':'gzip, deflate, sdch',
        'Accept-Language':'en',
        'Connection':'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
        'Referer': 'http://www.cnbeta.com/articles/tech/{}.htm'.format(sid),
        'Pragma':'no-cache',
        'X-Requested-With':'XMLHttpRequest',        
        'Cache-Control':'no-cache',
    }
    
    req = urllib.request.Request(url, headers = headers)
    req.set_proxy('{}:{}'.format(x['host'],x['port']), x['type'])
    try:
        res = urllib.request.urlopen(req, timeout = 10)
    except urllib.error.HTTPError as e:
        print(e.getcode())
        print(e.reason)
        print(e.geturl())
        print('-----------------')
        print(e.info())
        print(e.read())
        
    data = res.read().decode('utf-8')
    #print(data)
    json_data = json.loads(data)
    token = ''
    if json_data["message"] == "ok" and json_data["state"] == "success":
        token = json_data["result"]["token"]
        print(token)

    # get token 
    if token == '':
        print('get token failed...')
        return ''
    
    second_url = 'http://www.cnbeta.com/articles/do'
    #option = input('please input the option: y:good / n:bad > ');
    params =  urllib.parse.urlencode({ 'sid': sid, 'op': 'good' if option.lower()[0] == 'y' else 'bad', '_csrf': token})
    
    headers={
        'Host': 'www.cnbeta.com',
        'Cookie': cookie,
        'Accept':'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding':'gzip, deflate',
        'Accept-Language':'en',
        'Connection':'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
        'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8', 
        'Referer': 'http://www.cnbeta.com/articles/tech/{}.htm'.format(sid),
        'Content-Length': len(params),
    }
    #rint(headers, params, params.encode('utf-8'))
    req = urllib.request.Request(second_url, data = params.encode('utf-8'), method = 'POST', headers = headers) 
    req.set_proxy('{}:{}'.format(x['host'],x['port']), x['type'])
    
    try:
        res = urllib.request.urlopen(req, timeout = 5)
    except urllib.error.HTTPError as e:
        print(e.getcode())
        print(e.reason)
        print(e.geturl())
        print('-----------------')
        print(e.info())
        print(e.read()) 

    data = res.read().decode('utf-8')
    json_data = json.loads(data)
    
    if json_data['state'] == 'success':
        print('success')        
        print(json_data['result'])
        #success ,write ip to file
        filename = 'files/{}.txt'.format(sid)
        if os.path.exists(filename):
            with open(filename, 'a+', encoding = 'utf-8') as f:
                f.write('\n' + x['host'])
        else:
            with open(filename, 'w', encoding = 'utf-8') as f:
                f.write(x['host'])
    else:
        print('failed')
    
'''    
x={"from": "txt", "port": 80, "country": "CN", "anonymity": "transparent", "host": "124.133.230.254", "response_time": 1.5, "type": "http"}
start(x)

'''

def getProxy():
    g = GetProxy()
    g.init()
    g.load_plugins()
    g.grab_web_proxies()
    g.validate_web_proxies()
    return g.valid_proxies

def init():
    print('init....')
    nums = input('please input loop times > ')
    url = input('please input the special url > ')
    option = input('please input the option(y:good / n:bad) > ');
    
    #parse url
    temp_url = urllib.parse.unquote(url)
    real_url = urllib.parse.urlparse(temp_url)
    params = urllib.parse.parse_qsl(real_url.query)
    #print(params)
    sid = getSId(params)
    print(sid)
    
    if sid == '':
        print('url is incorrect...')
        return
    index = 1
    while (index <= int(nums)):
        # get used ip list
        print('get used ip list...')
        iplist = []
        filename = 'files/{}.txt'.format(sid)
        if os.path.exists(filename):
            with open(filename, 'r') as f:
                iplist = [ip.strip('\n') for ip in f.readlines()]

        # get proxy list
        print('get proxy list...')
        proxys = getProxy()
        print('proxy length: {}'.format(len(proxys)))
        
        for x in proxys:
            if x['host'] not in iplist and x['type'] == 'http':
                print(x)
                try:
                    start(x, url, option, sid)
                except Exception as e:
                    print(e)
        
        index = index + 1
        
        #sleep 10min
        print('sleep 5 min...')
        time.sleep(5 * 60)

init()