AWS api数据信息获取（boto3）

墨痕诉清风2024-09-01 19:03

GitHub - starsliao/TenSunS: 🦄后羿 - TenSunS(原ConsulManager)：基于Consul的运维平台：更优雅的Consul管理UI&多云与自建ECS/MySQL/Redis同步Prometheus/JumpServer&ECS/MySQL/Redis云监控指标采集&Blackbox站点监控维护&漏洞通知/资源到期余额告警&各类资源Grafana看板展示🦄后羿 - TenSunS(原ConsulManager)：基于Consul的运维平台：更优雅的Consul管理UI&多云与自建ECS/MySQL/Redis同步Prometheus/JumpServer&ECS/MySQL/Redis云监控指标采集&Blackbox站点监控维护&漏洞通知/资源到期余额告警&各类资源Grafana看板展示 - starsliao/TenSunShttps://github.com/starsliao/TenSunS

所有依赖（包含无用的，懒得不分了）

bash 复制代码 
Werkzeug==2.0.3
itsdangerous==2.0.1
flask==2.0.2
flask-restful==0.3.9
flask-cors==3.0.10
Flask-HTTPAuth==4.5.0
requests==2.27.1
Flask-APScheduler==1.12.3
xlrd==1.2.0
#pyDes==2.0.1
pycryptodome==3.14.1
beautifulsoup4==4.11.1
ldap3==2.9.1
loguru==0.6.0
huaweicloudsdkcore==3.1.11
huaweicloudsdkecs==3.1.11
huaweicloudsdkeps==3.1.11
huaweicloudsdkbss==3.1.11
huaweicloudsdkrds==3.1.11
huaweicloudsdkces==3.1.11
huaweicloudsdkdcs==3.1.11
alibabacloud_resourcemanager20200331==2.1.1
alibabacloud_ecs20140526==2.1.3
alibabacloud_rds20140815==2.1.2
alibabacloud_r_kvstore20150101==2.20.7
alibabacloud_bssopenapi20171214==2.0.6
aliyun-python-sdk-cms==7.0.32
tencentcloud-sdk-python-common==3.0.779
tencentcloud-sdk-python-cvm==3.0.779
tencentcloud-sdk-python-cdb==3.0.779
tencentcloud-sdk-python-dcdb==3.0.779
tencentcloud-sdk-python-billing==3.0.779
tencentcloud-sdk-python-monitor==3.0.779
tencentcloud-sdk-python-redis==3.0.779
boto3==1.23.10

源码

python 复制代码 
from Tea.exceptions import TeaException
import boto3

import sys,datetime,hashlib,math,traceback
from units import consul_kv,consul_svc
from units.cloud import sync_ecs,sync_rds,sync_redis,notify
from units.config_log import *

def exp(account,collect_days,notify_days,notify_amount):
    pass

def group(account):
    try:
        now = datetime.datetime.now().strftime('%m.%d/%H:%M')
        group_dict = {"1": "1"} # AWS没有资产组概念
        consul_kv.put_kv(f'ConsulManager/assets/awscloud/group/{account}',group_dict)
        count = len(group_dict)
        data = {'count':count,'update':now,'status':20000,'msg':f'同步资源组成功！总数：{count}'}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/group', data)
        logger.info(f'【JOB】===>awscloud_group {account} {data}')
    except TeaException as e:
        emsg = e.message.split('. ',1)[0]
        logger.error(f"【code:】{e.code}\n【message:】{emsg}\n{traceback.format_exc()}")
        data = consul_kv.get_value(f'ConsulManager/record/jobs/awscloud/{account}/group')
        if data == {}:
            data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg}
        else:
            data['update'] = f'失败{e.code}'
            data['msg'] = emsg
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/group', data)
    except Exception as e:
        logger.error(f'{e}\n{traceback.format_exc()}')
        data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/group', data)

def ecs(account,region,isextip=False):
    ak,sk = consul_kv.get_aksk('awscloud',account)
    now = datetime.datetime.now().strftime('%m.%d/%H:%M')
    group_dict = consul_kv.get_value(f'ConsulManager/assets/awscloud/group/{account}')  # {"1":"1"}

    ecs_dict = {}
    try:
        ec2 = boto3.client(
            "ec2",
            aws_access_key_id=ak,
            aws_secret_access_key=sk,
            region_name=region,
        )
        response = ec2.describe_instances()

        for reservations in response['Reservations']:
            for instances in reservations['Instances']:
                InstanceId = instances["InstanceId"]
                ecs_dict_temp = {InstanceId: {}}
                for tag in instances["Tags"]:
                    if tag["Key"] == "Name":
                        ecs_dict_temp[InstanceId]["name"] = tag["Value"]
                ecs_dict_temp[InstanceId]["group"] = "无"
                ecs_dict_temp[InstanceId]["ostype"] = "windows" if "win" in instances["PlatformDetails"].lower() else "linux"
                ecs_dict_temp[InstanceId]["status"] = instances["State"]["Name"]
                ecs_dict_temp[InstanceId]["region"] = region
                ecs_dict_temp[InstanceId]["ip"] = instances["PrivateIpAddress"]
                ecs_dict_temp[InstanceId]["cpu"] = f'{instances["CpuOptions"]["CoreCount"]*instances["CpuOptions"]["ThreadsPerCore"]}核'
                ecs_dict_temp[InstanceId]["mem"] = "0"
                ecs_dict_temp[InstanceId]["exp"] = "按量" # AWS仅按量 
                if isextip:
                    ecs_dict_temp[InstanceId]["ip"] = instances["PublicIpAddress"]
                ecs_dict.update(ecs_dict_temp)

        count = len(ecs_dict)
        off,on = sync_ecs.w2consul('awscloud',account,region,ecs_dict)
        data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'ECS同步成功！总数：{count}，开机：{on}，关机：{off}'}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/ecs/{region}', data)
        logger.info(f'【JOB】===>awscloud_ecs {account} {region} {data}')
    except TeaException as e:
        emsg = e.message.split('. ',1)[0]
        logger.error(f"【code:】{e.code}\n【message:】{emsg}\n{traceback.format_exc()}")
        data = consul_kv.get_value(f'ConsulManager/record/jobs/awscloud/{account}/ecs/{region}')
        if data == {}:
            data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg}
        else:
            data['update'] = f'失败{e.code}'
            data['msg'] = emsg
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/ecs/{region}', data)
    except Exception as e:
        logger.error(f'{e}\n{traceback.format_exc()}')
        data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/ecs/{region}', data)

def redis(account,region):
    ak,sk = consul_kv.get_aksk('awscloud',account)
    now = datetime.datetime.now().strftime('%m.%d/%H:%M')
    group_dict = consul_kv.get_value(f'ConsulManager/assets/awscloud/group/{account}')

    redis_dict = {}
    try:
        elasticache = boto3.client(
            'elasticache',
            aws_access_key_id=ak,
            aws_secret_access_key=sk,
            region_name=region,
        )
        response = elasticache.describe_cache_clusters()

        for rdb in response["CacheClusters"]:
            CacheClusterId = rdb["CacheClusterId"]
            redis_dict_temp = {CacheClusterId: {}}
            redis_dict_temp[CacheClusterId]["name"] = rdb["CacheClusterId"]
            redis_dict_temp[CacheClusterId]["domain"] = rdb.get("PrivateIp","null")
            redis_dict_temp[CacheClusterId]["ip"] = rdb.get("PrivateIp","null")
            redis_dict_temp[CacheClusterId]["port"] = 6379
            redis_dict_temp[CacheClusterId]["region"] = region
            redis_dict_temp[CacheClusterId]["group"] = rdb["ReplicationGroupId"]
            redis_dict_temp[CacheClusterId]["status"] = rdb["CacheClusterStatus"]
            redis_dict_temp[CacheClusterId]["itype"] = rdb["Engine"]
            redis_dict_temp[CacheClusterId]["ver"] = rdb["EngineVersion"]
            redis_dict_temp[CacheClusterId]["mem"] = "0"
            redis_dict_temp[CacheClusterId]["exp"] = "按量"
            redis_dict.update(redis_dict_temp)

        count = len(redis_dict)
        off,on = sync_redis.w2consul('awscloud',account,region,redis_dict)
        data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'redis同步成功！总数：{count}，开机：{on}，关机：{off}'}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/redis/{region}', data)
        logger.info(f'【JOB】===>awscloud_redis {account} {region} {data}')
    except TeaException as e:
        emsg = e.message.split('. ',1)[0]
        logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}")
        data = consul_kv.get_value(f'ConsulManager/record/jobs/awscloud/{account}/redis/{region}')
        if data == {}:
            data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg}
        else:
            data['update'] = f'失败{e.code}'
            data['msg'] = emsg
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/redis/{region}', data)
    except Exception as e:
        logger.error(f'{e}\n{traceback.format_exc()}')
        data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/redis/{region}', data)

def rds(account,region):
    ak,sk = consul_kv.get_aksk('awscloud',account)
    now = datetime.datetime.now().strftime('%m.%d/%H:%M')
    group_dict = consul_kv.get_value(f'ConsulManager/assets/awscloud/group/{account}')

    rds_dict = {}
    try:
        rds = boto3.client(
            "rds",
            aws_access_key_id=ak,
            aws_secret_access_key=sk,
            region_name=region,
        )
        response = rds.describe_db_instances()

        for db in response["DBInstances"]:
            DBInstanceId = db["DBInstanceIdentifier"]
            rds_dict_temp = {DBInstanceId: {}}
            rds_dict_temp[DBInstanceId]["name"] = db["DBInstanceIdentifier"]
            rds_dict_temp[DBInstanceId]["domain"] = db["Endpoint"]["Address"]
            rds_dict_temp[DBInstanceId]["ip"] = db["Endpoint"]["Address"]
            rds_dict_temp[DBInstanceId]["port"] = db["Endpoint"]["Port"]
            rds_dict_temp[DBInstanceId]["region"] = region
            rds_dict_temp[DBInstanceId]["group"] = db["DBClusterIdentifier"]
            rds_dict_temp[DBInstanceId]["status"] = db["DBInstanceStatus"]
            rds_dict_temp[DBInstanceId]["itype"] = db["Engine"]
            rds_dict_temp[DBInstanceId]["ver"] = db["EngineVersion"]
            rds_dict_temp[DBInstanceId]["exp"] = "按量"
            rds_dict_temp[DBInstanceId]["cpu"] = "0"
            rds_dict_temp[DBInstanceId]["mem"] = "0"
            rds_dict_temp[DBInstanceId]["disk"] = "0"
            rds_dict.update(rds_dict_temp)
 
        count = len(rds_dict)
        off,on = sync_rds.w2consul('awscloud',account,region,rds_dict)
        data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'rds同步成功！总数：{count}，开机：{on}，关机：{off}'}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/rds/{region}', data)
        logger.info(f'【JOB】===>awscloud_rds {account} {region} {data}')
    except TeaException as e:
        emsg = e.message.split('. ',1)[0]
        logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}")
        data = consul_kv.get_value(f'ConsulManager/record/jobs/awscloud/{account}/rds/{region}')
        if data == {}:
            data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg}
        else:
            data['update'] = f'失败{e.code}'
            data['msg'] = emsg
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/rds/{region}', data)
    except Exception as e:
        logger.error(f'{e}\n{traceback.format_exc()}')
        data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
        consul_kv.put_kv(f'ConsulManager/record/jobs/awscloud/{account}/rds/{region}', data)
相关推荐
小安运维日记17 分钟前
Linux云计算 |【第四阶段】NOSQL-DAY1
linux·运维·redis·sql·云计算·nosql
waterHBO24 分钟前
python 爬虫 selenium 笔记
爬虫·python·selenium
编程零零七1 小时前
Python数据分析工具(三)：pymssql的用法
开发语言·前端·数据库·python·oracle·数据分析·pymssql
AIAdvocate3 小时前
Pandas_数据结构详解
数据结构·python·pandas
小言从不摸鱼3 小时前
【AI大模型】ChatGPT模型原理介绍（下）
人工智能·python·深度学习·机器学习·自然语言处理·chatgpt
FreakStudio5 小时前
全网最适合入门的面向对象编程教程：50 Python函数方法与接口-接口和抽象基类
python·嵌入式·面向对象·电子diy
redcocal7 小时前
地平线秋招
python·嵌入式硬件·算法·fpga开发·求职招聘
artificiali7 小时前
Anaconda配置pytorch的基本操作
人工智能·pytorch·python
萌新求带啊QAQ7 小时前
腾讯云2024年数字生态大会开发者嘉年华（数据库动手实验）TDSQL-C初体验
云计算·腾讯云·tdsql-c
RaidenQ7 小时前
2024.9.13 Python与图像处理新国大EE5731课程大作业，索贝尔算子计算边缘，高斯核模糊边缘，Haar小波计算边缘
图像处理·python·算法·课程设计
热门推荐
01RAG 实践- Ollama+RagFlow 部署本地知识库02组基轨迹建模 GBTM的介绍与实现（Stata 或 R）032024年高教社杯数学建模国赛C题超详细解题思路分析04Coze扣子平台完整体验和实践（附国内和国际版对比）05苍穹外卖面试总结06【2024数模国赛赛题思路公开】国赛B题思路丨附可运行代码丨无偿自提07yolov8实战第五天——yolov8+ffmpg实时视频流检测并进行实时推流——（推流，保姆教学）08CCF-CSP认证考试 202406-3 文本分词 100分题解09【2024高教社杯全国大学生数学建模竞赛】B题 生产过程中的决策问题——解题思路 代码 论文10浏览器插件——ModHeader 的分享和学习