python爬虫之aiohttp多任务异步爬虫
爬取的flash服务如下：
from flask import Flask
import time
app = Flask(__name__)
@app.route('/bobo')
def index_bobo():
time.sleep(2)
return 'Hello bobo'
@app.route('/jay')
def index_jay():
time.sleep(2)
return 'Hello jay'
@app.route('/tom')
def index_tom():
time.sleep(2)
return 'Hello tom'
if __name__ == '__main__':
app.run(threaded=True)
运行启动flask服务后，多任务爬取代码如下：
#环境安装：pip install aiohttp
#使用该模块中的ClientSession
import requests
import asyncio
import time
import aiohttp
start = time.time()
urls = [
'http://127.0.0.1:5000/bobo','http://127.0.0.1:5000/jay','http://127.0.0.1:5000/tom'
]
async def get_page(url):
async with aiohttp.ClientSession() as session:
#get()、post():
#headers,params/data,proxy='http://ip:port'
async with await session.get(url) as response:
#text()返回字符串形式的响应数据
#read()返回二进制形式的响应数据
#json()返回的就是json对象
#注意：获取响应数据操作之前一定要使用await进行手动挂起
page_text = await response.text()
print(page_text)
tasks = []
for url in urls:
c = get_page(url)
task = asyncio.ensure_future(c)
tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
end = time.time()
print('总耗时：',end-start)