一、docker-compose.yml 安装 elasticsearch
version: '3.7'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0
container_name: elasticsearch
environment:
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- xpack.security.enabled=false # 禁用安全功能（开发环境建议关闭）
ulimits:
memlock:
soft: -1
hard: -1
ports:
- "9200:9200"
networks:
- esnet
kibana:
image: docker.elastic.co/kibana/kibana:8.14.0
container_name: kibana
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200 # 8.x 使用 ELASTICSEARCH_HOSTS
- SERVER_HOST=0.0.0.0 # 允许外部访问
- I18N_LOCALE=zh-CN # 汉化
ports:
- "5601:5601"
networks:
- esnet
depends_on:
- elasticsearch
filebeat:
image: docker.elastic.co/beats/filebeat:8.14.0
container_name: filebeat
user: root
volumes:
- /Users/jiajiamao/soft/elasticsearch/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
- /Users/jiajiamao/soft/elasticsearch/filebeat/log:/var/log
networks:
- esnet
depends_on:
- elasticsearch
networks:
esnet:
driver: bridge
二、挂载文件 kibana.yml
server.port: 5601
# 服务IP
server.host: "0.0.0.0"
# # ES
elasticsearch.hosts: ["http://localhost:9200"]
# # 汉化
i18n.locale: "zh-CN"
三、挂载文件 filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/micro-gateway/*.log
fields:
app_name: micro-gateway
- type: log
enabled: true
paths:
- /var/log/micro-user/*.log
fields:
app_name: micro-user
output.elasticsearch:
hosts: ["elasticsearch:9200"]
setup.kibana:
host: "kibana:5601"
# 服务IP
server.host: "0.0.0.0"
# ES
elasticsearch.hosts: ["http://elasticsearch:9200"]
# 汉化
i18n.locale: "zh-CN"
四、python后端开发环境搭建和配置
#集成 elasticsearch
pip install elasticsearch-dsl==8.14.0
pip install elasticsearch==8.14.0
# 查看是否安装完
pip show elasticsearch-dsl==8.14.0
pip show elasticsearch==8.14.0
五、启动 docker-compose 文件
#进入docker-compose.yml文件 目录
#执行
docker-compose up -d
六、验证 elasticsearch,kibana
#访问
http://localhost:9200/
#访问
http://localhost:5601/app/dev_tools#/console
七、后端构建目录
# 创建 operation 项目
django-admin startproject operation
#项目结构
├── db.sqlite3
├── dbOperations
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-313.pyc
│ │ ├── admin.cpython-313.pyc
│ │ ├── apps.cpython-313.pyc
│ │ ├── models.cpython-313.pyc
│ │ └── views.cpython-313.pyc
│ ├── admin.py
│ ├── apps.py
│ ├── migrations
│ │ ├── __init__.py
│ │ └── __pycache__
│ │ └── __init__.cpython-313.pyc
│ ├── models.py
│ ├── tests.py
│ └── views.py
├── deployments
│ ├── Consumers.py
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── Consumers.cpython-313.pyc
│ │ ├── __init__.cpython-313.pyc
│ │ ├── admin.cpython-313.pyc
│ │ ├── apps.cpython-313.pyc
│ │ ├── models.cpython-313.pyc
│ │ ├── routing.cpython-313.pyc
│ │ └── views.cpython-313.pyc
│ ├── admin.py
│ ├── apps.py
│ ├── migrations
│ │ ├── 0001_initial.py
│ │ ├── __init__.py
│ │ └── __pycache__
│ │ ├── 0001_initial.cpython-313.pyc
│ │ └── __init__.cpython-313.pyc
│ ├── models.py
│ ├── routing.py
│ ├── tests.py
│ └── views.py
├── log
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-313.pyc
│ │ ├── admin.cpython-313.pyc
│ │ ├── apps.cpython-313.pyc
│ │ ├── models.cpython-313.pyc
│ │ └── views.cpython-313.pyc
│ ├── admin.py
│ ├── apps.py
│ ├── migrations
│ │ ├── 0001_initial.py
│ │ ├── __init__.py
│ │ └── __pycache__
│ │ ├── 0001_initial.cpython-313.pyc
│ │ └── __init__.cpython-313.pyc
│ ├── models.py
│ ├── tests.py
│ └── views.py
├── manage.py
├── monitor
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-313.pyc
│ │ ├── admin.cpython-313.pyc
│ │ ├── apps.cpython-313.pyc
│ │ ├── models.cpython-313.pyc
│ │ ├── signals.cpython-313.pyc
│ │ ├── tasks.cpython-313.pyc
│ │ └── views.cpython-313.pyc
│ ├── admin.py
│ ├── apps.py
│ ├── migrations
│ │ ├── 0001_initial.py
│ │ ├── __init__.py
│ │ └── __pycache__
│ │ ├── 0001_initial.cpython-313.pyc
│ │ └── __init__.cpython-313.pyc
│ ├── models.py
│ ├── signals.py
│ ├── tasks.py
│ ├── tests.py
│ └── views.py
├── operation
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-313.pyc
│ │ ├── asgi.cpython-313.pyc
│ │ ├── settings.cpython-313.pyc
│ │ ├── urls.cpython-313.pyc
│ │ └── wsgi.cpython-313.pyc
│ ├── asgi.py
│ ├── common
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ └── __init__.cpython-313.pyc
│ │ ├── enum
│ │ │ ├── BuildProjectEnum.py
│ │ │ ├── EventTypesEnum.py
│ │ │ ├── OperatorTypeEnum.py
│ │ │ ├── ResponeCodeEnum.py
│ │ │ ├── __init__.py
│ │ │ └── __pycache__
│ │ │ ├── BuildProjectEnum.cpython-313.pyc
│ │ │ ├── EventTypesEnum.cpython-313.pyc
│ │ │ ├── OperatorTypeEnum.cpython-313.pyc
│ │ │ ├── ResponeCodeEnum.cpython-313.pyc
│ │ │ └── __init__.cpython-313.pyc
│ │ ├── exception
│ │ │ ├── BusinessException.py
│ │ │ ├── __init__.py
│ │ │ └── __pycache__
│ │ │ ├── BusinessException.cpython-313.pyc
│ │ │ └── __init__.cpython-313.pyc
│ │ └── utils
│ │ ├── CommonResult.py
│ │ ├── PageUtils.py
│ │ ├── Serializers.py
│ │ ├── __init__.py
│ │ └── __pycache__
│ │ ├── CommonResult.cpython-313.pyc
│ │ ├── PageUtils.cpython-313.pyc
│ │ ├── Serializers.cpython-313.pyc
│ │ └── __init__.cpython-313.pyc
│ ├── settings.py
│ ├── urls.py
│ └── wsgi.py
├── templates
└── user
├── __init__.py
├── __pycache__
│ ├── __init__.cpython-313.pyc
│ ├── admin.cpython-313.pyc
│ ├── apps.cpython-313.pyc
│ ├── models.cpython-313.pyc
│ └── views.cpython-313.pyc
├── admin.py
├── apps.py
├── migrations
│ ├── 0001_initial.py
│ ├── __init__.py
│ └── __pycache__
│ ├── 0001_initial.cpython-313.pyc
│ └── __init__.cpython-313.pyc
├── models.py
├── tests.py
└── views.py
七、后端deployments模块相关代码
####################################views.py#######################################
import asyncio
import datetime
import json
import logging
import os
import subprocess
import git
from django.http import JsonResponse, HttpResponse
from django.views.decorators.csrf import csrf_exempt
from git import Repo
from deployments.models import Deployment
from operation import settings
from operation.common.enum.BuildProjectEnum import BuildProjectEnum
from operation.common.enum.ResponeCodeEnum import ResponseCodeEnum
from operation.common.exception.BusinessException import BusinessException
from operation.common.utils.CommonResult import CommonResult
from operation.common.utils.PageUtils import paginate_queryset
from asgiref.sync import sync_to_async
from operation.es.documents.ElasticsearchLogger import ElasticsearchLogger
# Create your views here.
"""
****************************创建 deployments API视图
"""
"""
获取应用列表
"""
@csrf_exempt
def getDeploymentsList(request):
try:
if request.method == 'POST':
json_data = request.body
data = json.loads(json_data)
page = data.get('page', 1)
page_size = data.get('page_size', 10)
else:
raise BusinessException(ResponseCodeEnum.METHOD_ERROR.message, ResponseCodeEnum.METHOD_ERROR.code)
deployment_list = Deployment.objects.all().order_by('created_at')
deployment_lists, pagination_info = paginate_queryset(deployment_list, page, page_size)
deployment_lists = [Deployment.to_dict() for Deployment in deployment_lists]
logging.info(type(deployment_lists))
return JsonResponse(CommonResult.success_pagination(deployment_lists, pagination_info),
json_dumps_params={'ensure_ascii': False})
except BusinessException as e:
return JsonResponse(CommonResult.error(e.code, e.message), json_dumps_params={'ensure_ascii': False})
"""
增加应用
"""
@csrf_exempt
def addDeployment(request):
if request.method == "GET":
return BusinessException(ResponseCodeEnum.METHOD_ERROR.message, ResponseCodeEnum.METHOD_ERROR.code)
try:
json_data = request.body
data = json.loads(json_data)
name = data.get('name')
if name is None:
raise BusinessException(ResponseCodeEnum.PARAMS_ERROR.message, ResponseCodeEnum.PARAMS_ERROR.code)
# 保存应用
Deployment.objects.create(
name=name,
status=BuildProjectEnum.NOT_STARTED.code
)
return JsonResponse(CommonResult.success_data(None), json_dumps_params={'ensure_ascii': False})
except BusinessException as e:
return JsonResponse(CommonResult.error(e.code, e.message), json_dumps_params={'ensure_ascii': False})
"""
*********部署角本
报错: You cannot call this from an async context - use a thread or sync_to_async.
解决方案：1.使用 sync_to_async 2.使用 使用线程
"""
# 定义全局变量 progress
progress = 0
async def cicd_execute(project_id, project_name, send_progress, send_news):
global progress # 声明 progress 为全局变量
index_name = project_name
logger = ElasticsearchLogger(index_name)
try:
# 删除旧的日志索引
await logger.delete_index(index_name)
#记录初始化日志
await logger.log("开始初化项目", level=logging.INFO)
# 每次点击发布时，重置 progress 为0
progress = 0
# 1.初始化阶段
await init_project(project_id, project_name, send_progress,logger)
# 2.构建阶段
await build_and_deploy_project(project_id, project_name, send_progress, send_news,logger)
# 3.运行阶段
await run_project(project_id, project_name, send_progress,send_news)
except BusinessException as e:
logging.info(f"发生系统内部异常: {e}")
#记录es
error_message = f"发生系统内部异常: {e}"
await logger.log(error_message, level=logging.ERROR)
return
finally:
# 关闭 Elasticsearch 连接
await logger.close()
"""
1.初始化阶段
"""
async def init_project(project_id, project_name, send_progress,logger):
global progress # 声明 progress 为全局变量
try:
logging.info("=================开始初始化 项目 %s" % (project_name))
logging.info("初始化中......")
# 更新状态为 "初始化中" 记录进度值
progress += 20
await update_deployment_status(project_id, project_name, BuildProjectEnum.INITIALIZING.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.INITIALIZING.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
projectName = project_name
projectId = project_id
if projectName is None or projectId is None:
raise BusinessException(ResponseCodeEnum.PARAMS_ERROR.message, ResponseCodeEnum.PARAMS_ERROR.code)
# 环境变量配置
set_environment_variables()
# 加载仓库 拉取代码
await clone_or_pull_repo(project_name)
# 更新状态为 "初始化成功" 记录进度值
progress += 20
await update_deployment_status(project_id, project_name, BuildProjectEnum.INITIAL_SUCCESS.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.INITIAL_SUCCESS.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
#记录日志到 es
logging.info("初始化成功......")
#记录es
await logger.log("初始化成功......",level=logging.INFO)
except BusinessException as e:
logging.info("初始化失败......")
# 更新状态为 "初始化失败"
await update_deployment_status(project_id, project_name, BuildProjectEnum.INITIAL_FAILURE.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.INITIAL_FAILURE.code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.status_code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.message)
#记录es
error_message = f"发生系统内部异常: {e}"
await logger.log(error_message, level=logging.ERROR)
# 抛出异常
raise BusinessException(ResponseCodeEnum.INTERNAL_SERVER_ERROR.message,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.code)
"""
2.构建阶段
"""
async def build_and_deploy_project(project_id, project_name, send_progress, send_news,logger):
global progress
logging.info("=================构建 项目 %s" % (project_name))
logging.info("部署中......")
try:
# 更新状态为 "部署中" 记录进度值 60
progress += 20
await update_deployment_status(project_id, project_name, BuildProjectEnum.DEPLOYMENT_IN.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.DEPLOYMENT_IN.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
projectName_image = project_name + "-image"
projectName_dir = os.path.join(settings.PROJECT_REPO_DIR, project_name)
await clean_old_containers(projectName_image)
await clean_old_images(projectName_image)
os.chdir(projectName_dir)
logging.info(f"当前工作目录: {os.getcwd()}")
# 执行 maven 打包
await execute_maven_build(project_id,project_name,send_news,logger)
logging.info("mvn clean install -DskipTests 执行完成......")
# 构建 image
await build_docker_image(projectName_image,send_news,logger)
logging.info("docker build -t , projectName_image . 执行完成")
# 更新状态为 "部署成功" 记录进度值 80
progress += 20
await update_deployment_status(project_id, project_name, BuildProjectEnum.DEPLOYMENT_SUCCESS.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.DEPLOYMENT_SUCCESS.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
logging.info("部署成功......")
except Exception as ex:
# 更新状态为 "部署失败"
logging.info("部署失败......")
await update_deployment_status(project_id, project_name, BuildProjectEnum.DEPLOYMENT_FAILURE.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.DEPLOYMENT_FAILURE.code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.status_code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.message)
# 记录es
error_message = f"发生系统内部异常: {ex}"
await logger.log(error_message, level=logging.ERROR)
# 抛出异常
raise BusinessException(ResponseCodeEnum.INTERNAL_SERVER_ERROR.message,ResponseCodeEnum.INTERNAL_SERVER_ERROR.code)
"""
3.运行阶段
"""
async def run_project(project_id, project_name, send_progress,send_news):
global progress
logging.info("=================启动 项目 %s" % (project_name))
logging.info("启动中......")
try:
# 更新状态为 "启动中" 记录进度值 70
progress += 10
await update_deployment_status(project_id, project_name, BuildProjectEnum.STARTING.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.STARTING.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
projectName_image = project_name + "-image"
await run_docker_container(projectName_image,send_news)
# 更新状态为 "启动成功" 记录进度值 80
progress += 10
await update_deployment_status(project_id, project_name, BuildProjectEnum.STARTED_SECCESS.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.STARTED_SECCESS.code, ResponseCodeEnum.SUCCESS.status_code,
ResponseCodeEnum.SUCCESS.message)
logging.info("启动成功......")
except Exception as ex:
# 更新状态为 "启动失败"
logging.info("启动失败......")
await update_deployment_status(project_id, project_name, BuildProjectEnum.STARTED_FAILURE.code, progress)
# 发送事件
await send_progress(progress, BuildProjectEnum.STARTED_FAILURE.code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.status_code,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.message)
# 抛出异常
raise BusinessException(ResponseCodeEnum.INTERNAL_SERVER_ERROR.message,
ResponseCodeEnum.INTERNAL_SERVER_ERROR.code)
"""
加载环境变量
"""
def set_environment_variables():
logging.info("加载环境变量......")
os.environ['PATH'] = settings.MAVEN_PATH + os.environ['PATH']
jdk_path = settings.JDK_PATH
os.environ['JAVA_HOME'] = jdk_path
os.environ['PATH'] = f"{jdk_path}/bin:{os.environ['PATH']}"
logging.info("环境变量加载成功......")
"""
加载仓库 拉取代码
"""
async def clone_or_pull_repo(project_name):
logging.info("拉取仓库代码......")
project_repo_url = settings.PROJECT_REPO_URL
project_repo_dir = settings.PROJECT_REPO_DIR
if not os.path.exists(project_repo_dir):
await sync_to_async(Repo.clone_from)(project_repo_url, project_repo_dir)
else:
project_repo = await sync_to_async(Repo)(project_repo_dir)
await sync_to_async(project_repo.remotes.origin.pull)()
logging.info("拉取仓库代码成功......")
"""
清理旧容器
"""
async def clean_old_containers(projectName_image):
cid = await sync_to_async(subprocess.check_output)(
["docker", "ps", "-a", "-q", "--filter", f"ancestor={projectName_image}"])
if cid:
# 去除末尾的换行符并转换为字符串
cid_str = cid.decode().strip()
logging.info("=================cid: %s" % (cid_str))
logging.info(f"清理旧容器 docker rm -f {cid_str}")
await sync_to_async(subprocess.run)(["docker", "rm", "-f", cid_str])
"""
清理旧镜像
"""
async def clean_old_images(projectName_image):
iid = await sync_to_async(subprocess.check_output)(["docker", "images", "-q", projectName_image])
if iid:
# 去除末尾的换行符并转换为字符串
iid_str = iid.strip()
logging.info("=================iid: %s" % (iid_str))
logging.info(f"清理旧镜像 docker rmi -f {iid_str}")
await sync_to_async(subprocess.run)(["docker", "rmi", "-f", iid_str])
"""
maven 打包构建
"""
async def execute_maven_build(project_id,project_name,send_news,logger):
# 使用 asyncio.create_subprocess_exec 创建异步进程
processObject = await asyncio.create_subprocess_exec(
'mvn', 'clean', 'install', '-DskipTests',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
logging.info("发送maven打包构建执行日志......")
#发送ES
await logger.log("发送maven打包构建执行日志......",level=logging.INFO)
await deployment_log_output2(processObject, send_news, ResponseCodeEnum.SUCCESS,logger)
"""
构建镜像
"""
async def build_docker_image(projectName_image, send_news,logger):
logging.info(f"=================构建镜像 docker build -t {projectName_image} .")
# 使用 asyncio.create_subprocess_exec 来异步执行命令
imageProcessObject = await asyncio.create_subprocess_exec(
'docker', 'build', '-t', projectName_image, '.',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
logging.info("发送构建镜像执行日志......")
await deployment_log_output2(imageProcessObject, send_news, ResponseCodeEnum.SUCCESS,logger)
"""
运行 docker
"""
async def run_docker_container(projectName_image, send_news):
logging.info(f"=================运行 docker run -d -p 8084:8084 {projectName_image} .")
containerRunProcessObject = await asyncio.create_subprocess_exec(
'docker', 'run', '-d', '-p', '8084:8084', projectName_image, '.',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
logging.info("发送启动docker执行日志......")
# 获取容器 ID
stdout, stderr = await containerRunProcessObject.communicate()
container_id = stdout.decode('utf-8').strip()
# 异步运行 docker logs 命令
containerRunlogprocess = await asyncio.create_subprocess_exec(
'docker', 'logs', '-f', '-n', '100', container_id,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
# 异步读取容器日志
asyncio.create_task(deployment_log_output(containerRunlogprocess, send_news, ResponseCodeEnum.SUCCESS))
"""
更新model
"""
async def update_deployment_status(project_id, project_name, status, progress):
await sync_to_async(Deployment.update_deployment_model)(project_id, project_name, status=status, progress=progress)
"""
读取mvn 执行命令
定义一个异步的 函数 deployment_log_output
async def:协程（协程是由用户程序控制的） 可以暂停和恢复执行
部署日志模块记录
processObject: 进程对象
send_news: 处理消息的方法
buildProjectEnum:构建项目的 Enum类
responseCodeEnum: 响应Enum类
progress: 进度值
"""
async def deployment_log_output(processObject, send_news, responseCodeEnum: ResponseCodeEnum):
try:
# 使用 async for 循环读取日志
async for line in processObject.stdout:
if not line:
break
# 发送日志
await send_news(line.decode('utf-8'), responseCodeEnum)
logging.info(f"发送日志信息成功: %s" % (line.decode('utf-8')))
finally:
# 等待子进程执行完毕
await processObject.wait()
async def deployment_log_output2(processObject, send_news, responseCodeEnum: ResponseCodeEnum,logger):
try:
# 使用 async for 循环读取日志
async for line in processObject.stdout:
if not line:
break
# 发送日志
await send_news(line.decode('utf-8'), responseCodeEnum)
logging.info(f"发送日志信息成功: %s" % (line.decode('utf-8')))
#记录es
await logger.log(line.decode('utf-8'),level=logging.INFO)
finally:
# 等待子进程执行完毕
await processObject.wait()
"""
查询日志
"""
async def getIndexName(request):
if request.method == "POST":
raise BusinessException(ResponseCodeEnum.METHOD_ERROR.message, ResponseCodeEnum.METHOD_ERROR.code)
logger = None
try:
indexName = request.GET.get('indexName')
if indexName is None:
raise BusinessException(ResponseCodeEnum.PARAMS_ERROR.message, ResponseCodeEnum.PARAMS_ERROR.code)
try:
logger = ElasticsearchLogger(indexName)
# 定义查询条件（Elasticsearch 查询 DSL）
query = {
"query": {
"match_all": {} # 查询所有数据
},
"from": 0, # 从第几条开始
"size": 100 # 返回多少条数据
}
result = await logger.search_by_index(indexName, query)
logging.info(f"result:{str(result)}")
return JsonResponse(CommonResult.success_asyn_data(result),
json_dumps_params={'ensure_ascii': False})
except Exception as e:
"""
f 是 f-string 的缩写，是一种字符串格式化方式。它允许你在字符串中直接嵌入变量或表达式，用大括号 {} 包裹起来，Python 会自动将其替换为对应的值
"""
logging.error(f"系统异常: {str(e)}")
raise BusinessException(ResponseCodeEnum.INTERNAL_SERVER_ERROR.message, ResponseCodeEnum.INTERNAL_SERVER_ERROR.code)
except BusinessException as buessException:
logging.error(f"系统异常: {str(buessException)}")
return JsonResponse(CommonResult.error(buessException.code, buessException.message), json_dumps_params={'ensure_ascii': False})
finally:
if logger is not None:
await logger.close()
# #检查 logger 变量是否在当前作用域中定义。如果 logger 已经定义，则执行 logger.close()。
# if 'logger' in locals():
# logger.close()
################################Consumers.py#############################
import os
from typing import Any
import django
import logging
from channels.generic.websocket import AsyncWebsocketConsumer
import json
from deployments.views import cicd_execute
from operation.common.enum.ResponeCodeEnum import ResponseCodeEnum
# 手动设置 DJANGO_SETTINGS_MODULE
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'operation.settings')
# 配置 Django
django.setup()
class ChatConsumer(AsyncWebsocketConsumer):
async def connect(self):
logging.info("connect")
await self.accept()
async def disconnect(self, close_code):
pass
"""
Exception inside application: You cannot call this from an async context - use a thread or sync_to_async.
Traceback (most recent call last):
这个错误提示表明你在异步上下文中调用了同步代码，这在 Django Channels 中是不允许的。
你需要将同步代码转换为异步代码，
或者使用 sync_to_async 包装同步代码。
"""
async def receive(self, text_data):
text_data_json = json.loads(text_data)
id = text_data_json['id']
project_name = text_data_json['projectName']
# 调用CI/CD执行函数
await self.channel_layer.send(
self.channel_name,
{
'type': 'execute_cicd',
'id': id,
'project_name': project_name,
}
)
async def execute_cicd(self, event):
id = event['id']
project_name = event['project_name']
# 调用CI/CD执行函数
await cicd_execute(id, project_name, self.send_progress,self.send_news)
"""
发送 进度
"""
async def send_progress(self, progress,status,code,message):
await self.send(text_data=json.dumps({
'type': 'progress',
'code': code,
'message': message,
'data': {
'progress': progress, #进度值
'status': status # 状态
}
}))
"""
发送 执行日志
: 类型注解，用于明确指定 responseCodeEnum 参数的类型是 ResponseCodeEnum
"""
async def send_news(self,logInfo: Any,responseCodeEnum:ResponseCodeEnum):
await self.send(text_data=json.dumps({
'type': 'news',
'code': responseCodeEnum.code,
'message': responseCodeEnum.message,
'data': {
'logInfo':logInfo #日志信息
}
}))
#################################routing.py#################################
#websocket
# your_app_name/routing.py
from django.urls import path
from . import Consumers
websocket_urlpatterns = [
path('ws/cicd_progress/', Consumers.ChatConsumer.as_asgi())
]
#################################settings.py#######################################
# Application definition
#它列出了所有在你的项目中启用的应用（apps）。每个应用可以提供特定的功能或服务
#通过在 INSTALLED_APPS 中列出这些应用，Django会在启动时自动加载它们，
# 并使它们的功能在你的项目中可用。
# 这样，你可以利用这些内置功能来快速开发和扩展你的应用
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'log',
'monitor',
'user',
'dbOperations',
'deployments',
'resources',
'channels'
]
ASGI_APPLICATION = 'operation.asgi.application'
WSGI_APPLICATION = 'operation.wsgi.application'
# elasticsearch
ELASTICSEARCH_DSL = {
'default': {
'hosts': 'http://localhost:9200' # Elasticsearch 地址
},
}
#日志
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'loggers': {
# 'django': {
# 'handlers': ['console'],
# 'level': 'INFO',
# 'propagate': False,
# },
'django.db.backends': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': False,
},
},
}
################################urls.py################################
urlpatterns = [
path('deployments/getDeploymentsList', deployment_views.getDeploymentsList,name='deployments_list'),
path('deployments/addDeployment', deployment_views.addDeployment, name='add_deployment'),
path('deployments/getIndexName', deployment_views.getIndexName, name='get_index_name'),
]
##########################CommonResult.py##########################
#工具类
import asyncio
from typing import Any, Optional
from operation.common.enum.ResponeCodeEnum import ResponseCodeEnum
class CommonResult:
"""
res: ResponseCodeEnum 这个定义表示 res 参数的类型是 ResponseCodeEnum，
即 res 必须是 ResponseCodeEnum 的枚举值之一。这种定义方式有助于提高代码的类型安全性和可读性。
"""
def __init__(self, res: ResponseCodeEnum, data: Any, pagination: Optional[dict] = None):
self.ResponseCodeEnum = ResponseCodeEnum
self.data = data
self.pagination = pagination
"""
表态方法定义 直接通过 CommonResult.success 类名.静态方法名
@staticmethod 不能访问或修改类或实例的属性。
"""
@staticmethod
def success_pagination(data: Any, pagination: Optional[dict] = None):
return {
'code': ResponseCodeEnum.SUCCESS.status_code,
'message': ResponseCodeEnum.SUCCESS.message,
'data': data, # 将 QuerySet 序列化为 JSON,
'pagination': pagination
}
@staticmethod
def error(code,message):
return {
'code': code,
'message': message,
'data': None, # 将 QuerySet 序列化为 JSON,
'pagination': None
}
@staticmethod
def success_data(data: Any):
return {
'code': ResponseCodeEnum.SUCCESS.status_code,
'message': ResponseCodeEnum.SUCCESS.message,
'data': data, # 将 QuerySet 序列化为 JSON,
}
"""
协程对象：result 中可能包含了一个异步函数返回的协程对象（例如 await 调用的结果）。
JSON 序列化：JsonResponse 在内部会调用 json.dumps()，而 json.dumps() 无法处理协程对象。
如果 result 包含协程对象，需要提前处理或转换为同步数据。(await)
"""
@staticmethod
def success_asyn_data(data: Any):
# 检查是否为协程对象
if asyncio.iscoroutine(data):
data = None # 或者根据需求处理协程对象
return {
'code': ResponseCodeEnum.SUCCESS.status_code,
'message': ResponseCodeEnum.SUCCESS.message,
'data': data,
}
###############################ResponseCodeEnum.py###############################
#工具类
from enum import Enum
class ResponseCodeEnum(Enum):
SUCCESS = (200, "操作成功！")
PARAMS_ERROR = (400, "参数解析失败,请核对参数！")
UNAUTHORIZED = (401, "未认证（签名错误）")
FORBIDDEN = (402, "请求错误") # 返回失败业务公共code
MEDIA_TYPE_ERROR = (403, "不支持的媒体异常,请核对contentType！")
URL_REQ_NULL = (404, "请求路径不存在")
METHOD_ERROR = (405, "不支持当前请求方法,请核对请求方法！")
INTERNAL_SERVER_ERROR = (500, "服务器内部错误！") # 系统异常公共code
NULL_POINTER_ERROR = (600, "请核对必填字段是否为空！")
NUMBER_FORMAT_ERROR = (601, "数据类型不一致！")
PARAMS_TYPE_ERROR = (602, "参数类型错误！")
TIMEOUT_ERROR = (603, "token失效连接超时！")
TIMEOUT_EXPIRE_ERROR = (604, "token登录过期！")
TOKEN_ILLEGAL = (605, "非法token！")
USER_LOGIN_FAIL = (1001,"用户登录失败!")
DATA_NONE = (1002, "记录不存在!")
################################################################
RESOURCES_IP_EXIST = (2001,"资源IP已存在!")
def __init__(self, code, message):
self.code = code
self.message = message
#@property 是 Python 中用于将类的方法转换为属性访问的装饰器。
# 使用 @property 装饰器，你可以像访问属性一样访问方法，而不需要调用它
@property
def status_code(self):
return self.code
@property
def error_message(self):
return self.message
#<class 'enum.EnumType'> 枚举类
# print(type(ResponseCodeEnum))
#<enum 'ResponseCodeEnum'> SUCCESS 是 ResponseCodeEnum 枚举类的一个实例。
# print(type(ResponseCodeEnum.SUCCESS))
八、启动django项目 （需要支持WebSocket或其他异步通信，使用daphne启动）
"""
和项目交互基本上都是基于这个文件，一般都是在终端输入python3 manage.py [子命令]
manage.py输入python3 manage.py help查看更多
启动项目 python3 manage.py runserver
创建app模块 python3 manage.py startapp app
生成迁移文件，描述如何将模型更改应用到数据库中 python3 manage.py makemigrations
将这些文件迁移应用到数据库中 python3 manage.py migrate
如果你需要支持WebSocket或其他异步通信，使用daphne operation.asgi:application。
如果你只需要简单的HTTP服务，使用python manage.py runserver。
export DJANGO_SETTINGS_MODULE=your_project_name.settings
"""
#安装Daphne
pip install daphne
#终端输入
daphne operation.asgi:application
九、前端部分代码
<template>
<div class="app-container">
<el-card shadow="always">
<el-form ref="searchForm" :inline="true" :model="searchMap" style="margin-top: 20px">
<el-form-item>
<el-button type="primary" icon="el-icon-search" @click="searchLog('searchForm')">搜索</el-button>
<el-button type="primary" icon="el-icon-clear" @click="resetForm('searchForm')">重置</el-button>
</el-form-item>
</el-form>
</el-card>
<el-card shadow="always">
<template>
<el-button size="mini" icon="el-icon-plus" type="primary" @click="openAddDrawer()">
新增
</el-button>
</template>
</el-card>
<el-row :gutter="24">
<el-col :span="24">
<el-card shadow="always">
<div>
<el-table ref="multipleTable" v-loading="listLoading" :data="getDeploymentListDto" border fit highlight-current-row style="width: 100%;" class="tb-edit">
<el-table-column prop="id" label="应用id" width="180px" align="center" />
<el-table-column prop="name" label="应用名称" width="180px" align="center" />
<el-table-column prop="status" label="状态" width="180px" align="center" :formatter="statusFormatter" />
<el-table-column prop="created_at" label="创建时间" width="180px" align="center" />
<el-table-column prop="actions" label="操作">
<template slot-scope="scope">
<el-button type="text" @click="redeploy(scope.row)">重新部署</el-button>
</template>
</el-table-column>
<!-- <el-table-column label="部署进度">
<template slot-scope="scope">
<el-progress :percentage="scope.row.progress" v-if="scope.row.progress"></el-progress>
</template>
</el-table-column> -->
<el-table-column label="部署进度">
<template slot-scope="scope">
<div style="display: flex; align-items: center;">
<el-progress v-if="scope.row.progress" :percentage="scope.row.progress" style="flex: 1;" />
<!-- <el-button type="text" style="margin-left: 10px;" @click="viewLogs(scope.row)">查看日志</el-button> -->
<el-button type="text" style="margin-left: 10px;" @click="viewLogs(scope.row)">
查看日志
</el-button>
</div>
</template>
</el-table-column>
</el-table>
</div>
<div class="block">
<el-pagination :current-page="currentPage" :page-sizes="[5, 10, 15, 20]" :page-size="5" layout="total, sizes, prev, pager, next, jumper" :total="total" @size-change="handleSizeChange" @current-change="handleCurrentChange" />
</div>
</el-card>
</el-col>
</el-row>
<!--新增项目 Drawer 层-->
<el-drawer title="增加应用" :visible.sync="addDrawerVisible" :direction="direction" size="50%">
<el-card shadow="always">
<el-form ref="addProjectForm" :model="addProjectForm" status-icon :rules="rules" label-width="100px" class="demo-ruleForm">
<el-form-item label="应用名称" prop="name">
<el-input v-model="addProjectForm.name" placeholder="请输入应用名称" />
</el-form-item>
<el-form-item>
<el-button type="primary" @click="addProjectSubmitForm('addProjectForm')">
保存
</el-button>
</el-form-item>
</el-form>
</el-card>
</el-drawer>
<!-- 日志弹框 -->
<el-dialog :visible.sync="viewLogDrawerVisible" width="70%" class="log-dialog" :fullscreen="isFullscreen" :show-close="false" :modal="false" ref="logDialog">
<!-- 自定义标题区域 -->
<template #title>
<div class="dialog-header" @mousedown="startDrag">
<span>发布日志</span>
<div class="header-buttons">
<el-button type="text" :icon="isFullscreen ? 'el-icon-copy-document' : 'el-icon-full-screen'" @click="toggleFullscreen" />
<el-button type="text" icon="el-icon-close" @click="closeDialog" />
</div>
</div>
</template>
<!-- 日志内容区域 -->
<div class="log-container">
<!-- 左侧项目名区域 -->
<div class="project-info">
<h3>当前发布项目</h3>
<p @click="fetchProjectDetail">{{ currentProject }}</p>
</div>
<!-- 右侧日志区域 -->
<el-card shadow="always" class="log-card">
<el-scrollbar ref="logScrollbar" style="height: 800px">
<div class="log-content" v-html="logContent" />
</el-scrollbar>
</el-card>
</div>
</el-dialog>
</div>
</template>
<script>
import {
getDeploymentsList,
// viewCICD,
addDeployment,
getIndexName
} from '@/api/deployment/deployment-request'
// import {
// message
// } from 'rhea-promise'
// import {
// ElScrollbar
// } from 'element-ui'
export default {
name: 'DeploymentPage',
data() {
return {
getDeploymentListDto: [], // 数据传给list,列表渲染的数据
total: 0,
listLoading: true,
dialogVisible: false,
currentPage: 1,
addDrawerVisible: false,
viewLogDrawerVisible: false,
direction: 'rtl',
isFullscreen: false, // 控制全屏状态
isDragging: false, // 是否正在拖拽
startX: 0, // 拖拽起始 X 坐标
startY: 0, // 拖拽起始 Y 坐标
dialogOffsetX: 0, // Dialog 的 X 偏移量
dialogOffsetY: 0, // Dialog 的 Y 偏移量
logContent: '',
currentProject: "",
jsonstr: {
'id': '',
'projectName': ''
},
addProjectForm: {
name: ''
},
rules: {
name: [{
required: true,
message: '请输入应用名称',
trigger: 'blur'
}]
},
params: {
page: 1, // 当前页码
page_size: 5 // 每页显示数目
},
searchMap: {},
socket: null
}
},
mounted() {
this.getDeploymentsList()
},
beforeDestroy() {
if (this.socket) {
this.socket.close()
}
},
methods: {
handleSizeChange(val) {
console.log(`每页 ${val} 条`)
this.params.page_size = val
this.getDeploymentsList()
},
handleCurrentChange(val) {
console.log(`当前页: ${val}`)
this.params.page = val
this.getDeploymentsList()
},
// 格式化 列
statusFormatter(row, column, cellValue) {
switch (cellValue) {
case '0':
return '未启动'
case '1':
return '初始化中'
case '2':
return '初始化成功'
case '3':
return '初始失败'
case '4':
return '部署中'
case '5':
return '部署成功'
case '6':
return '部署失败'
case '7':
return '启动中'
case '8':
return '启动成功'
case '9':
return '启动失败'
default:
return '未知状态'
}
},
// 重置功能, element ui 提供的功能
resetForm(formName) {
console.log(this.$refs[formName].resetFields)
this.$refs[formName].resetFields()
this.getDeploymentsList()
},
searchLog(formName) {
console.log(this.searchMap)
this.getDeploymentsList()
},
// 新增项目
openAddDrawer() {
this.addDrawerVisible = true // 显示Drawer
},
viewLogs(row) {
// 这里可以添加查看日志的逻辑
// console.log('查看日志:', row);
// // 例如，可以打开一个新的对话框或跳转到日志页面
// this.$message.info(`查看日志: ${row.name}`);
this.currentProject = row.name
this.viewLogDrawerVisible = true
},
// 操作日志列表 ajax 请求
getDeploymentsList() {
// 目标需求：在历史查询列表页面中加入查询的转圈的loading加载动画。
this.dataLoading = true
console.log('请求参数:' + this.params)
getDeploymentsList(this.params).then((res) => {
console.log('响应:', res.data.data)
this.getDeploymentListDto = res.data.data
this.total = res.data.pagination.total
setTimeout(() => { // 超过指定超时时间 关闭查询的转圈的loading加载动画
this.listLoading = false
}, 1.5 * 1000)
})
},
addProjectSubmitForm(formName) {
this.$refs[formName].validate((valid) => {
if (valid) {
this.addProjectInfo()
} else {
console.log('error submit!!')
this.getDeploymentsList()
return false
}
})
},
// 增加应用
addProjectInfo() {
this.dataLoading = true
addDeployment(this.addProjectForm).then((res) => {
console.log(res)
this.addDrawerVisible = false // 关闭抽屉
this.getDeploymentsList()
this.$refs.addOrgForm.resetFields()
// this.$router.push('/organizationList'); // 假设列表页的路由路径是 /list
setTimeout(() => {
this.listLoading = false
}, 1.5 * 1000)
})
},
redeploy(row) {
this.socket = new WebSocket('ws://localhost:8000/ws/cicd_progress/')
this.jsonstr.id = row.id
this.jsonstr.projectName = row.name
this.socket.onopen = () => {
this.socket.send(JSON.stringify(this.jsonstr))
}
this.socket.onmessage = (event) => {
const data = JSON.parse(event.data)
if (data.type === 'progress') {
// 处理进度消息
if (data.code !== 200) {
// 处理错误消息
row.status = data.data.status // 更新当前状态
alert(data.data.status)
this.getDeploymentsList()
} else {
// 更新进度条
const progress = data.data.progress // 假设服务器返回的进度数据在progress字段中
row.progress = progress // 更新进度条的百分比
row.status = data.data.status // 更新当前状态
this.getDeploymentsList()
}
} else if (data.type === 'news') {
// 处理日志消息
this.logContent += `<p>${data.data.logInfo}</p>` // 追加日志
this.$nextTick(() => {
this.scrollToBottom() // 每次更新日志后滚动到底部
})
} else {
console.error('Unknown message type:', data.type)
}
}
this.socket.onclose = () => {
console.log('WebSocket closed')
row.status = '5' // 假设部署成功后状态为5
}
this.socket.onerror = (error) => {
console.error('WebSocket error:', error)
row.status = '6' // 假设部署失败后状态为6
this.$message.error('WebSocket 连接错误') // 显示错误消息
}
},
scrollToBottom() {
const scrollbar = this.$refs.logScrollbar.$refs.wrap
scrollbar.scrollTop = scrollbar.scrollHeight
},
toggleFullscreen() {
this.isFullscreen = !this.isFullscreen; // 切换全屏状态
},
closeDialog() {
this.viewLogDrawerVisible = false;
},
// 开始拖拽
startDrag(event) {
this.isDragging = true;
this.startX = event.clientX;
this.startY = event.clientY;
document.addEventListener("mousemove", this.onDrag);
document.addEventListener("mouseup", this.stopDrag);
},
// 拖拽中
onDrag(event) {
if (this.isDragging) {
const offsetX = event.clientX - this.startX;
const offsetY = event.clientY - this.startY;
this.dialogOffsetX += offsetX;
this.dialogOffsetY += offsetY;
this.startX = event.clientX;
this.startY = event.clientY;
// 更新 Dialog 位置
const dialog = this.$refs.logDialog.$el.querySelector(".el-dialog");
if (dialog) {
dialog.style.transform = `translate(${this.dialogOffsetX}px, ${this.dialogOffsetY}px)`;
}
}
},
// 停止拖拽
stopDrag() {
this.isDragging = false;
document.removeEventListener("mousemove", this.onDrag);
document.removeEventListener("mouseup", this.stopDrag);
},
async fetchProjectDetail() {
// 目标需求：在历史查询列表页面中加入查询的转圈的loading加载动画。
this.dataLoading = true
getIndexName(this.currentProject).then((res) => {
console.log('响应:', res.data)
// 处理日志数据
this.logContent = res.data.data
.map(line => line.trim()) // 去除每行的前后空白
.join('<br>'); // 用 <br> 标签连接每一行
setTimeout(() => { // 超过指定超时时间 关闭查询的转圈的loading加载动画
this.listLoading = false
}, 1.5 * 1000)
})
},
}
}
</script>
<style scoped>
.log-drawer {
background-color: #f5f7fa;
/* 设置背景色 */
}
.log-container {
display: flex;
height: 100%;
}
.project-info {
width: 200px;
/* 左侧区域宽度 */
padding: 16px;
background-color: #ffffff;
/* 左侧背景色 */
border-right: 1px solid #e4e7ed;
/* 右侧分割线 */
}
.project-info h3 {
margin: 0 0 12px 0;
font-size: 16px;
color: #333;
}
.project-info p {
margin: 0;
font-size: 14px;
color: #666;
}
.log-card {
flex: 1;
/* 右侧区域自适应宽度 */
border-radius: 0;
/* 去掉圆角 */
background-color: #000;
/* 黑色背景 */
color: #fff;
/* 白色文字 */
}
.log-content {
font-family: 'Courier New', Courier, monospace;
/* 使用等宽字体 */
font-size: 14px;
/* 字体大小 */
line-height: 1.6;
/* 行高 */
padding: 16px;
/* 内边距 */
}
.log-content p {
margin: 0 0 8px 0;
/* 段落间距 */
color: #00ff00;
/* 绿色日志文字 */
}
.log-content p:last-child {
margin-bottom: 0;
/* 最后一个段落去掉底部间距 */
}
.dialog-header {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 16px;
font-weight: bold;
}
.header-buttons .el-button {
padding: 0;
font-size: 18px;
}
.header-buttons {
display: flex;
align-items: center;
gap: 10px;
}
.log-dialog {
transition: all 0.3s ease;
}
</style>
十、验证
#新增项目
#点击重新发布
#点击项目名
micro-admin
#通过 index-name 从elasticsearch中读取最新一次打包记录
