python实现http get post download
使用requests, 加上重试机制,超时机制.
python
#!/usr/bin/python3
# -*- coding: UTF-8 -*-
import requests
import sys
import json
import os
import logging
import time
def httpGet(urlStr, headers = None, params = None, data = None, isRaiseNone200=False):
print('url:' + urlStr)
print('params:', params)
MAX_RETRY_COUT = 5
# 加个重试次数,可根据需要定制哪些情况下进行重试,如果不重试就把raise的注释去掉
for retTryPos in range(MAX_RETRY_COUT):
try:
# 这里timeout要加,设置连接与访问超时时间,否则可能遇到http卡死挂住的情况
res = requests.request('get', urlStr, params=params, headers=headers, data=data, timeout=(15, 20))
content = res.text
print(content)
if res.status_code != 200:
if isRaiseNone200:
raise ValueError('http不是200')
else:
print('重试')
time.sleep(5)
continue
else:
return content
except requests.exceptions.ConnectionError as e:
logging.error('网络连接异常: ', e)
print('重试')
time.sleep(5)
#raise
except requests.exceptions.Timeout as e:
logging.error('连接超时: ', e)
print('重试')
time.sleep(5)
#raise
except requests.exceptions.RequestException as e:
logging.error('请求异常: ', e)
time.sleep(5)
#raise
except requests.exceptions.HTTPError as e:
logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
time.sleep(5)
#raise
except ValueError as e:
logging.error('响应解析异常: ', e)
print('重试')
time.sleep(5)
#raise
raise ValueError('超出重试次数')
def httpDownload(urlStr, saveFile):
print('download url:' + urlStr)
savePath = os.path.dirname(saveFile)
if not os.path.exists(savePath):
print('mkdir:', savePath)
os.makedirs(str(savePath))
r = requests.get(urlStr, timeout=(15, 30))
if r.status_code != 200:
print('http返回:', r.content)
print('http code:', r.status_code)
raise ValueError('code不是200')
tmpFile = saveFile + '.tmp'
with open(tmpFile, 'wb') as f:
f.write(r.content)
f.close()
os.rename(tmpFile, saveFile)
def httpPost(urlStr, headers, postData):
print('url:' + urlStr)
try:
token = getToken()
datas = json.dumps(postData)
rc = requests.post(url=urlStr, data=datas, headers=headers, timeout=(8, 20))
print('post:' + str(datas))
content = (rc.text)
print(content)
if rc.status_code != 200:
raise ValueError('code不是200')
return content
except requests.exceptions.ConnectionError as e:
logging.error('网络连接异常: ', e)
time.sleep(5)
#raise
except requests.exceptions.Timeout as e:
logging.error('连接超时: ', e)
time.sleep(5)
#raise
except requests.exceptions.RequestException as e:
logging.error('请求异常: ', e)
time.sleep(5)
#raise
except requests.exceptions.HTTPError as e:
logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
time.sleep(5)
#raise
except ValueError as e:
logging.error('响应解析异常: ', e)
time.sleep(5)
#raise
raise ValueError('超出重试次数')
if __name__ == '__main__':
httpGet('http://www.baidu.com')
作者:帅得不敢出门 csdn原创谢绝转载收录