python实现http get pos download

python实现http get post download

使用requests, 加上重试机制,超时机制.

#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import requests
import sys
import json
import os
import logging
import time

def httpGet(urlStr, headers = None, params = None, data = None, isRaiseNone200=False):
    print('url:' + urlStr)
    print('params:', params)

    MAX_RETRY_COUT = 5
    # 加个重试次数,可根据需要定制哪些情况下进行重试,如果不重试就把raise的注释去掉
    for retTryPos in range(MAX_RETRY_COUT):
        try:
            # 这里timeout要加,设置连接与访问超时时间,否则可能遇到http卡死挂住的情况
            res = requests.request('get', urlStr, params=params, headers=headers, data=data, timeout=(15, 20))
            content = res.text
            print(content)

            if res.status_code != 200:
                if isRaiseNone200:
                    raise ValueError('http不是200')
                else:
                    print('重试')
                    time.sleep(5)
                    continue
            else:
                return content
        except requests.exceptions.ConnectionError as e:
            logging.error('网络连接异常: ', e)
            print('重试')
            time.sleep(5)
            #raise
        except requests.exceptions.Timeout as e:
            logging.error('连接超时: ', e)
            print('重试')
            time.sleep(5)
            #raise
        except requests.exceptions.RequestException as e:
            logging.error('请求异常: ', e)
            time.sleep(5)
            #raise
        except requests.exceptions.HTTPError as e:
            logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
            time.sleep(5)
            #raise
        except ValueError as e:
            logging.error('响应解析异常: ', e)
            print('重试')
            time.sleep(5)
            #raise

    raise ValueError('超出重试次数')

def httpDownload(urlStr, saveFile):
    print('download url:' + urlStr)
    savePath = os.path.dirname(saveFile)
    if not os.path.exists(savePath):
        print('mkdir:', savePath)
        os.makedirs(str(savePath))

    r = requests.get(urlStr, timeout=(15, 30))
    if r.status_code != 200:
        print('http返回:', r.content)
        print('http code:', r.status_code)
        raise ValueError('code不是200')

    tmpFile = saveFile + '.tmp'
    with open(tmpFile, 'wb') as f:
        f.write(r.content)
        f.close()
        os.rename(tmpFile, saveFile)

def httpPost(urlStr, headers, postData):
    print('url:' + urlStr)
    try:
        token = getToken()
        datas = json.dumps(postData)
        rc = requests.post(url=urlStr, data=datas, headers=headers, timeout=(8, 20))
        print('post:' + str(datas))
        content = (rc.text)
        print(content)
        if rc.status_code != 200:
            raise ValueError('code不是200')
        return content
    except requests.exceptions.ConnectionError as e:
        logging.error('网络连接异常: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.Timeout as e:
        logging.error('连接超时: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.RequestException as e:
        logging.error('请求异常: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.HTTPError as e:
        logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
        time.sleep(5)
        #raise
    except ValueError as e:
        logging.error('响应解析异常: ', e)
        time.sleep(5)
        #raise

    raise ValueError('超出重试次数')


if __name__ == '__main__':
    httpGet('http://www.baidu.com')

作者:帅得不敢出门 csdn原创谢绝转载收录