python爬取网页接口数据,以yearning为例

模拟登陆获取token,传token到对应的接口获取数据,下载到csv里面

python 复制代码
import getpass
import os
import requests
import time
import csv
from datetime import datetime

class Yearning:

    def __init__(self):
        self.session = requests.Session()
        self.host = 'http://yearning.xxxx.com:8000'
        self.token_file = './token.txt'
        if not os.path.exists(self.token_file):
            open(self.token_file, 'w').close()
        file = open(self.token_file, 'r')
        token = file.read()
        file.close()
        self.authorization = ''
        if token:
            self.authorization = f'Bearer {token}'

    def query(self, database, source, sql, print_sql=False):
        if print_sql:
            print(sql)
        uri = f'{self.host}/api/v2/query/results'
        body = {
            'data_base': database,
            'source': source,
            'sql': sql
        }
        try:
            result_data = self.get_result_data(uri, body, False)
            if result_data and 'data' in result_data and result_data['data']:
                return result_data['data']
            return []
        except Exception as e:
            print(f'query exception! database={database}, source={source}, sql={sql}')
            print(e)
            return []


    def login(self, username, password):
        uri = f'{self.host}/ldap'
        headers = {
            'Content-Type': 'application/json; charset=UTF-8',
            'cookie': '',
        }
        body = {
            'username': username,
            'password': password
        }
        try:
            result_data = self.get_result_data(uri, body, False)
            if result_data and 'token' in result_data:
                token = result_data['token']
                self.authorization = f'Bearer {token}'
                file = open(self.token_file, 'w')
                file.truncate()
                file.write(token)
                file.close()
            else:
                print(f'login fail!')
        except Exception as e:
            print(f'login error! e={e}')

    def get_result_data(self, uri, body, isGet):
        headers = {
            'Content-Type': 'application/json; charset=UTF-8',
            'cookie': '',
            'Authorization': self.authorization
        }
        if isGet:
            response = requests.get(uri, headers=headers)
        else:
            response = requests.post(uri, json=body, headers=headers)
        result_data = response.json()
        if result_data == 'missing or malformed jwt' or result_data == 'Token is expired' or result_data == '':
            print('登录授权已失效,请重新登录!')
            username = input('账号:')
            password = getpass.getpass('密码:')
            # 可以hardcode账号密码
            #username = ''
            #password = ''
            self.login(username, password)
            return self.get_result_data(uri, body, isGet)
        if 'code' in result_data and result_data['code'] == 1200 and 'payload' in result_data:
            return result_data['payload']
        else:
            print(f'其他异常,休眠1s!{result_data}')
            time.sleep(1)
            return None


yearning = Yearning()


def export():
    sql = "select   count(*), driver_id from   order_info group by driver_id"
    db = "order"
    source = "order_readonly"
    # 获取当前时间
    now = datetime.now()
    # 格式化时间为年月日时分秒字符串,例如:"2023-04-01 14:30:45"
    datetime_str = now.strftime("%Y%m%d%H%M%S")
    #windows的换行符是\r\n 会导致有空行,需要将换行符修改为\n,lineterminator='\n'
    save_file = csv.writer(open(f'res/data-{datetime_str}.csv', 'w'),lineterminator='\n')
    data_list = yearning.query(db, source, sql, True)
    data = data_list[0]
    key_list = [key for key in data]
    save_file.writerow(key_list)
    print(key_list)
    for data in data_list:
        list = [data[key].strip() for key in key_list]
        print(list)
        save_file.writerow(list)






if __name__ == '__main__':
    export()
相关推荐
XiaoLeisj32 分钟前
【JavaEE初阶 — 多线程】单例模式 & 指令重排序问题
java·开发语言·java-ee
API快乐传递者36 分钟前
淘宝反爬虫机制的主要手段有哪些?
爬虫·python
励志成为嵌入式工程师2 小时前
c语言简单编程练习9
c语言·开发语言·算法·vim
师太,答应老衲吧2 小时前
SQL实战训练之,力扣:2020. 无流量的帐户数(递归)
数据库·sql·leetcode
捕鲸叉2 小时前
创建线程时传递参数给线程
开发语言·c++·算法
A charmer2 小时前
【C++】vector 类深度解析:探索动态数组的奥秘
开发语言·c++·算法
Peter_chq2 小时前
【操作系统】基于环形队列的生产消费模型
linux·c语言·开发语言·c++·后端
阡之尘埃3 小时前
Python数据分析案例61——信贷风控评分卡模型(A卡)(scorecardpy 全面解析)
人工智能·python·机器学习·数据分析·智能风控·信贷风控
Channing Lewis3 小时前
salesforce case可以新建一个roll up 字段,统计出这个case下的email数量吗
数据库·salesforce
记录成长java4 小时前
ServletContext,Cookie,HttpSession的使用
java·开发语言·servlet