Created
March 11, 2019 10:52
-
-
Save gsw945/d3e7fd3c2916368abbff7572c412de8a to your computer and use it in GitHub Desktop.
concurrency requests demo
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| import time | |
| import logging | |
| import threading | |
| from multiprocessing.dummy import Pool as ThreadPool | |
| from multiprocessing import Manager | |
| try: | |
| from http.client import RemoteDisconnected | |
| except ImportError: | |
| from httplib import BadStatusLine as RemoteDisconnected | |
| import requests | |
| from colorama import init; init() | |
| from colorama import Fore, Back, Style | |
| ''' | |
| Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. | |
| Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. | |
| Style: DIM, NORMAL, BRIGHT, RESET_ALL | |
| ''' | |
| def request_get(url, headers=None): | |
| '''发起请求''' | |
| if headers is None: | |
| headers = { | |
| 'User-Agent': 'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' | |
| } | |
| return requests.get(url, headers=headers) | |
| def download_item(args): | |
| '''下载每一条数据''' | |
| (link, share, lock, data_len, retries) = args | |
| req_time = 0 | |
| while retries > req_time: | |
| req_time += 1 | |
| try: | |
| resp = request_get(link) | |
| except RemoteDisconnected: | |
| print('RemoteDisconnected: retry[{0}/{1}]'.format(req_time, retries)) | |
| continue | |
| else: | |
| break | |
| item_status = False | |
| if resp.ok: | |
| # 模拟耗时 | |
| import random | |
| time.sleep(random.randint(0, 15)) | |
| # TODO: 从响应结果中保存数据 | |
| item_status = True | |
| else: | |
| print(resp.status_code, end=' -> ') | |
| print(link) | |
| # 加锁-开始修改共享变量 | |
| lock.acquire() | |
| # 记录某个地址是否成功 | |
| share[link] = item_status | |
| # 已处理的数量 | |
| share_len = len(share) | |
| # 释放锁 | |
| lock.release() | |
| time.sleep(0.5) | |
| # 显示处理进度 | |
| prints = ( | |
| Fore.RED + Back.GREEN + Style.BRIGHT, | |
| share_len, data_len, '成功' if item_status else '失败', link, | |
| Style.RESET_ALL | |
| ) | |
| print('{0}{1:2}/{2:2}[{3}] -> [{4}]{5}'.format(*prints)) | |
| def download(links, concurrency=None, retries=None): | |
| '''下载''' | |
| # 并发数 | |
| if concurrency is None: | |
| concurrency = 4 | |
| # 出错后重试次数 | |
| if retries is None: | |
| retries = 3 | |
| # 进程间通信共享数据 | |
| manager = Manager() | |
| # 共享数据 | |
| share = manager.dict() | |
| # 线程锁 | |
| lock = threading.Lock() | |
| # 线程池 | |
| tpool = ThreadPool(concurrency) | |
| # 数据长度 | |
| data_len = len(links) | |
| # 预处理每条数据 | |
| data = list(map(lambda link: (link, share, lock, data_len, retries), links)) | |
| # 启动任务 | |
| tresult_proxy = tpool.map_async(download_item, data) | |
| # 关闭线程池 | |
| tpool.close() | |
| # 等到线程池所有任务完成 | |
| tpool.join() | |
| # 获取结果 | |
| return tresult_proxy.get() | |
| def setup_log(): | |
| '''日志-记录requests请求''' | |
| logging.basicConfig() | |
| logging.getLogger().setLevel(logging.DEBUG) | |
| requests_log = logging.getLogger("requests.packages.urllib3") | |
| requests_log.setLevel(logging.DEBUG) | |
| requests_log.propagate = True | |
| def main(debug=False, concurrency=4, retries=3): | |
| if debug: | |
| setup_log() | |
| demo_links = [ | |
| 'https://segmentfault.com/a/{0}'.format(item) | |
| for item in [ | |
| '1190000018454271', | |
| '1190000000340291', | |
| '1190000009754256', | |
| '1190000004872691', | |
| '1190000016411674', | |
| '1190000010176121', | |
| ] | |
| ] + [ | |
| 'https://image-static.segmentfault.com/350/727/3507276754-5c85a71337ab7_articlex', | |
| 'https://image-static.segmentfault.com/175/364/175364187-5aaf400d928c4_articlex', | |
| 'https://image-static.segmentfault.com/282/296/2822965528-5c85a713130c6_articlex', | |
| 'https://image-static.segmentfault.com/223/778/2237785828-5c85a712f373c_articlex', | |
| 'https://image-static.segmentfault.com/419/490/4194902075-5c85a71301f18_articlex', | |
| 'https://image-static.segmentfault.com/136/251/1362516142-5c85a712c6b36_articlex', | |
| 'https://image-static.segmentfault.com/195/261/1952614898-5c85a712d1c05_articlex', | |
| 'https://image-static.segmentfault.com/138/000/1380002042-5c85a712cc829_articlex', | |
| 'https://image-static.segmentfault.com/104/873/1048730447-5c85a712cb892_articlex', | |
| 'https://image-static.segmentfault.com/338/118/3381181618-5c85a712dea18_articlex', | |
| 'https://image-static.segmentfault.com/146/358/1463582500-5c85a7133091b_articlex' | |
| ] | |
| download(demo_links, concurrency=concurrency, retries=retries) | |
| if __name__ == '__main__': | |
| main(debug=True, concurrency=5) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| import time | |
| import logging | |
| from multiprocessing import ( | |
| Pool, Manager, | |
| freeze_support | |
| ) | |
| try: | |
| from http.client import RemoteDisconnected | |
| except ImportError: | |
| from httplib import BadStatusLine as RemoteDisconnected | |
| import requests | |
| from colorama import init; init() | |
| from colorama import Fore, Back, Style | |
| ''' | |
| Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. | |
| Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. | |
| Style: DIM, NORMAL, BRIGHT, RESET_ALL | |
| ''' | |
| def request_get(url, headers=None): | |
| '''发起请求''' | |
| if headers is None: | |
| headers = { | |
| 'User-Agent': 'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' | |
| } | |
| return requests.get(url, headers=headers) | |
| def download_item(args): | |
| '''下载每一条数据''' | |
| (link, share, lock, data_len, debug, retries) = args | |
| if debug: | |
| setup_log() | |
| req_time = 0 | |
| while retries > req_time: | |
| req_time += 1 | |
| try: | |
| resp = request_get(link) | |
| except RemoteDisconnected: | |
| print('RemoteDisconnected: retry[{0}/{1}]'.format(req_time, retries)) | |
| continue | |
| else: | |
| break | |
| item_status = False | |
| if resp.ok: | |
| # 模拟耗时 | |
| import random | |
| time.sleep(random.randint(0, 15)) | |
| # TODO: 从响应结果中保存数据 | |
| item_status = True | |
| else: | |
| print(resp.status_code, end=' -> ') | |
| print(link) | |
| # 加锁-开始修改共享变量 | |
| lock.acquire() | |
| # 记录某个地址是否成功 | |
| share[link] = item_status | |
| # 已处理的数量 | |
| share_len = len(share) | |
| # 释放锁 | |
| lock.release() | |
| time.sleep(0.5) | |
| # 显示处理进度 | |
| prints = ( | |
| Fore.RED + Back.GREEN + Style.BRIGHT, | |
| share_len, data_len, '成功' if item_status else '失败', link, | |
| Style.RESET_ALL | |
| ) | |
| print('{0}{1:2}/{2:2}[{3}] -> [{4}]{5}'.format(*prints)) | |
| def download(links, debug=False, concurrency=None, retries=None): | |
| '''下载''' | |
| # 并发数 | |
| if concurrency is None: | |
| concurrency = 4 | |
| # 出错后重试次数 | |
| if retries is None: | |
| retries = 3 | |
| # 进程间通信共享数据 | |
| manager = Manager() | |
| # 共享数据 | |
| share = manager.dict() | |
| # 进程锁 | |
| lock = manager.Lock() | |
| # 进程池 | |
| pool = Pool(concurrency) | |
| # 数据长度 | |
| data_len = len(links) | |
| # 预处理每条数据 | |
| data = list(map(lambda link: (link, share, lock, data_len, debug, retries), links)) | |
| # 启动任务 | |
| tresult_proxy = pool.map_async(download_item, data) | |
| # 关闭进程池 | |
| pool.close() | |
| # 等到进程池所有任务完成 | |
| pool.join() | |
| # 获取结果 | |
| return tresult_proxy.get() | |
| def setup_log(): | |
| '''日志-记录requests请求''' | |
| logging.basicConfig() | |
| logging.getLogger().setLevel(logging.DEBUG) | |
| requests_log = logging.getLogger("requests.packages.urllib3") | |
| requests_log.setLevel(logging.DEBUG) | |
| requests_log.propagate = True | |
| def main(debug=False, concurrency=4, retries=3): | |
| demo_links = [ | |
| 'https://segmentfault.com/a/{0}'.format(item) | |
| for item in [ | |
| '1190000018454271', | |
| '1190000000340291', | |
| '1190000009754256', | |
| '1190000004872691', | |
| '1190000016411674', | |
| '1190000010176121', | |
| ] | |
| ] + [ | |
| 'https://image-static.segmentfault.com/350/727/3507276754-5c85a71337ab7_articlex', | |
| 'https://image-static.segmentfault.com/175/364/175364187-5aaf400d928c4_articlex', | |
| 'https://image-static.segmentfault.com/282/296/2822965528-5c85a713130c6_articlex', | |
| 'https://image-static.segmentfault.com/223/778/2237785828-5c85a712f373c_articlex', | |
| 'https://image-static.segmentfault.com/419/490/4194902075-5c85a71301f18_articlex', | |
| 'https://image-static.segmentfault.com/136/251/1362516142-5c85a712c6b36_articlex', | |
| 'https://image-static.segmentfault.com/195/261/1952614898-5c85a712d1c05_articlex', | |
| 'https://image-static.segmentfault.com/138/000/1380002042-5c85a712cc829_articlex', | |
| 'https://image-static.segmentfault.com/104/873/1048730447-5c85a712cb892_articlex', | |
| 'https://image-static.segmentfault.com/338/118/3381181618-5c85a712dea18_articlex', | |
| 'https://image-static.segmentfault.com/146/358/1463582500-5c85a7133091b_articlex' | |
| ] | |
| download(demo_links, debug=debug, concurrency=concurrency, retries=retries) | |
| if __name__ == '__main__': | |
| freeze_support() | |
| main(debug=True, concurrency=5) |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
多进程+多线程 混合 requests 发请求示例: gsw945/simple-ddos.py