python实现http get pos download

python实现http get post download

使用requests, 加上重试机制,超时机制.

python 复制代码
#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import requests
import sys
import json
import os
import logging
import time

def httpGet(urlStr, headers = None, params = None, data = None, isRaiseNone200=False):
    print('url:' + urlStr)
    print('params:', params)

    MAX_RETRY_COUT = 5
    # 加个重试次数,可根据需要定制哪些情况下进行重试,如果不重试就把raise的注释去掉
    for retTryPos in range(MAX_RETRY_COUT):
        try:
            # 这里timeout要加,设置连接与访问超时时间,否则可能遇到http卡死挂住的情况
            res = requests.request('get', urlStr, params=params, headers=headers, data=data, timeout=(15, 20))
            content = res.text
            print(content)

            if res.status_code != 200:
                if isRaiseNone200:
                    raise ValueError('http不是200')
                else:
                    print('重试')
                    time.sleep(5)
                    continue
            else:
                return content
        except requests.exceptions.ConnectionError as e:
            logging.error('网络连接异常: ', e)
            print('重试')
            time.sleep(5)
            #raise
        except requests.exceptions.Timeout as e:
            logging.error('连接超时: ', e)
            print('重试')
            time.sleep(5)
            #raise
        except requests.exceptions.RequestException as e:
            logging.error('请求异常: ', e)
            time.sleep(5)
            #raise
        except requests.exceptions.HTTPError as e:
            logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
            time.sleep(5)
            #raise
        except ValueError as e:
            logging.error('响应解析异常: ', e)
            print('重试')
            time.sleep(5)
            #raise

    raise ValueError('超出重试次数')

def httpDownload(urlStr, saveFile):
    print('download url:' + urlStr)
    savePath = os.path.dirname(saveFile)
    if not os.path.exists(savePath):
        print('mkdir:', savePath)
        os.makedirs(str(savePath))

    r = requests.get(urlStr, timeout=(15, 30))
    if r.status_code != 200:
        print('http返回:', r.content)
        print('http code:', r.status_code)
        raise ValueError('code不是200')

    tmpFile = saveFile + '.tmp'
    with open(tmpFile, 'wb') as f:
        f.write(r.content)
        f.close()
        os.rename(tmpFile, saveFile)

def httpPost(urlStr, headers, postData):
    print('url:' + urlStr)
    try:
        token = getToken()
        datas = json.dumps(postData)
        rc = requests.post(url=urlStr, data=datas, headers=headers, timeout=(8, 20))
        print('post:' + str(datas))
        content = (rc.text)
        print(content)
        if rc.status_code != 200:
            raise ValueError('code不是200')
        return content
    except requests.exceptions.ConnectionError as e:
        logging.error('网络连接异常: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.Timeout as e:
        logging.error('连接超时: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.RequestException as e:
        logging.error('请求异常: ', e)
        time.sleep(5)
        #raise
    except requests.exceptions.HTTPError as e:
        logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')
        time.sleep(5)
        #raise
    except ValueError as e:
        logging.error('响应解析异常: ', e)
        time.sleep(5)
        #raise

    raise ValueError('超出重试次数')


if __name__ == '__main__':
    httpGet('http://www.baidu.com')

作者:帅得不敢出门 csdn原创谢绝转载收录

相关推荐
TF男孩5 小时前
ARQ:一款低成本的消息队列,实现每秒万级吞吐
后端·python·消息队列
该用户已不存在10 小时前
Mojo vs Python vs Rust: 2025年搞AI,该学哪个?
后端·python·rust
站大爷IP12 小时前
Java调用Python的5种实用方案:从简单到进阶的全场景解析
python
用户83562907805117 小时前
从手动编辑到代码生成:Python 助你高效创建 Word 文档
后端·python
侃侃_天下17 小时前
最终的信号类
开发语言·c++·算法
c8i17 小时前
python中类的基本结构、特殊属性于MRO理解
python
echoarts18 小时前
Rayon Rust中的数据并行库入门教程
开发语言·其他·算法·rust
liwulin050618 小时前
【ESP32-CAM】HELLO WORLD
python
Aomnitrix18 小时前
知识管理新范式——cpolar+Wiki.js打造企业级分布式知识库
开发语言·javascript·分布式
Doris_202318 小时前
Python条件判断语句 if、elif 、else
前端·后端·python