MySQL 元数据库 Docker Compose 部署 Apache DolphinScheduler 3.1.9 集群实战

背景

今天我们将深入探讨一个备受关注的话题:如何使用

MySQL元数据库,通过 Docker Compose 快速部署

Apache DolphinScheduler 3.1.9 集群。

大家都知道,DolphinScheduler 官方提供的 Docker Compose 部署方案默认使用的是 PostgreSQL 作为元数据存储。这主要是由于 MySQL 的 GPLv2 协议与 Apache License 2.0 之间存在不兼容性,导致官方无法直接提供基于 MySQL 的版本。然而,在实际生产环境中,很多团队更倾向于使用MySQL作为数据库,因为它在运维、工具支持和社区生态方面具有广泛的优势。

本篇文章将为您揭秘,如何在官方方案的基础上进行巧妙调整,让您的 DolphinScheduler 集群成功跑在 MySQL元数据库上。

下载镜像

复制代码
docker pull apache/dolphinscheduler-master:3.1.9

docker pull apache/dolphinscheduler-worker:3.1.9

docker pull apache/dolphinscheduler-tools:3.1.9

docker pull apache/dolphinscheduler-api:3.1.9

docker pull apache/dolphinscheduler-alert-server:3.1.9

docker pull bitnami/zookeeper:3.7.1

下载MySQL驱动

复制代码
wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-j-8.0.33.zip

unzip -q mysql-connector-j-8.0.33.zip

cp mysql-connector-j-8.0.33/mysql-connector-j-8.0.33.jar .

准备镜像文件

master、worker、api、alert-server使用的Dockerfile

复制代码
# 基于DolphinScheduler官方镜像

ARG SERVICE=api

FROM apache/dolphinscheduler-${SERVICE}:3.1.9


# 将MySQL JDBC驱动拷贝到DolphinScheduler的lib目录

# DolphinScheduler通常会从其lib目录加载JDBC驱动

COPY mysql-connector-j-8.0.33.jar /opt/dolphinscheduler/libs/

tools使用的Dockerfile

复制代码
# 基于DolphinScheduler官方镜像

ARG SERVICE=tools

FROM apache/dolphinscheduler-${SERVICE}:3.1.9


# 将MySQL JDBC驱动拷贝到DolphinScheduler的lib目录

# DolphinScheduler通常会从其lib目录加载JDBC驱动

COPY mysql-connector-j-8.0.33.jar /opt/dolphinscheduler/tools/libs/

编译新镜像

复制代码
docker build --build-arg SERVICE=master -t apache/dolphinscheduler-master:3.1.9-mysql .

docker build --build-arg SERVICE=worker -t apache/dolphinscheduler-worker:3.1.9-mysql .

docker build --build-arg SERVICE=tools -t apache/dolphinscheduler-tools:3.1.9-mysql .

docker build --build-arg SERVICE=api -t apache/dolphinscheduler-api:3.1.9-mysql .

docker build --build-arg SERVICE=alert-server -t apache/dolphinscheduler-alert-server:3.1.9-mysql .

修改docker-compose.yaml

注释掉postgresql service,增加mysql service

复制代码
# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements.  See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership.  The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.


services:

#dolphinscheduler-postgresql:

#  image: bitnami/postgresql:15.2.0

#  ports:

#    - "5432:5432"

#  profiles: ["all", "schema"]

#  environment:

#    POSTGRESQL_USERNAME: root

#    POSTGRESQL_PASSWORD: root

#    POSTGRESQL_DATABASE: dolphinscheduler

#  volumes:

#    - dolphinscheduler-postgresql:/bitnami/postgresql

#  healthcheck:

#    test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"]

#    interval: 5s

#    timeout: 60s

#    retries: 120

#  networks:

#    - dolphinscheduler

# MySQL 数据库服务 (元数据库)

dolphinscheduler-mysql:

image:
mysql:8.0

container_name:
dolphinscheduler-mysql

profiles:
 [
"all"
, 
"schema"
]

environment:

MYSQL_ROOT_PASSWORD:
${MYSQL_ROOT_PASSWORD:-root}

MYSQL_DATABASE:
${MYSQL_DATABASE:-dolphinscheduler}

volumes:

-
dolphinscheduler-mysql:/var/lib/mysql

ports:

-
"3306:3306"
# 映射MySQL端口到宿主机,供其他服务器的Worker访问

healthcheck:

test:
 [
"CMD"
, 
"mysqladmin"
, 
"ping"
, 
"-h"
, 
"localhost"
, 
"-u"
, 
"${MYSQL_USERNAME:-root}"
, 
"-p${MYSQL_PASSWORD:-root}"
] 
# 健康检查,使用 .env 中的用户名密码

interval:
5s

timeout:
60s

retries:
120

networks:

-
dolphinscheduler


dolphinscheduler-zookeeper:

image:
bitnami/zookeeper:3.7.1

profiles:
 [
"all"
]

environment:

ALLOW_ANONYMOUS_LOGIN:
"yes"

ZOO_4LW_COMMANDS_WHITELIST:
srvr,ruok,wchs,cons

ports:

-
"2181:2181"

volumes:

-
dolphinscheduler-zookeeper:/bitnami/zookeeper

healthcheck:

test:
 [
"CMD"
, 
"bash"
, 
"-c"
, 
"cat < /dev/null > /dev/tcp/127.0.0.1/2181"
]

interval:
5s

timeout:
60s

retries:
120

networks:

-
dolphinscheduler


dolphinscheduler-schema-initializer:

image:
${HUB}/dolphinscheduler-tools:${TAG}

env_file:
.env

profiles:
 [
"schema"
]

command:
 [ 
tools/bin/upgrade-schema.sh
 ]

depends_on:

dolphinscheduler-mysql:

condition:
service_healthy

volumes:

-
dolphinscheduler-logs:/opt/dolphinscheduler/logs

-
dolphinscheduler-shared-local:/opt/soft

-
/dolphinscheduler:/dolphinscheduler

networks:

-
dolphinscheduler


dolphinscheduler-api:

image:
${HUB}/dolphinscheduler-api:${TAG}

ports:

-
"12345:12345"

-
"25333:25333"

profiles:
 [
"all"
]

env_file:
.env

healthcheck:

test:
 [ 
"CMD"
, 
"curl"
, 
"http://localhost:12345/dolphinscheduler/actuator/health"
 ]

interval:
30s

timeout:
5s

retries:
3

depends_on:

dolphinscheduler-zookeeper:

condition:
service_healthy

volumes:

-
dolphinscheduler-logs:/opt/dolphinscheduler/logs

-
dolphinscheduler-shared-local:/opt/soft

-
/dolphinscheduler:/dolphinscheduler

networks:

-
dolphinscheduler


dolphinscheduler-alert:

image:
${HUB}/dolphinscheduler-alert-server:${TAG}

profiles:
 [
"all"
]

env_file:
.env

healthcheck:

test:
 [ 
"CMD"
, 
"curl"
, 
"http://localhost:50053/actuator/health"
 ]

interval:
30s

timeout:
5s

retries:
3

depends_on:

dolphinscheduler-zookeeper:

condition:
service_healthy

volumes:

-
dolphinscheduler-logs:/opt/dolphinscheduler/logs

networks:

-
dolphinscheduler


dolphinscheduler-master:

image:
${HUB}/dolphinscheduler-master:${TAG}

profiles:
 [
"all"
]

env_file:
.env

environment:

#增加java参数设置

JAVA_OPTS:
-server
-Duser.timezone=${SPRING_JACKSON_TIME_ZONE}
-Xms8g
-Xmx8g
-Xmn4g
-XX:+PrintGCDetails
-Xloggc:gc.loo

g
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof

healthcheck:

test:
 [ 
"CMD"
, 
"curl"
, 
"http://localhost:5679/actuator/health"
 ]

interval:
30s

timeout:
5s

retries:
3

depends_on:

dolphinscheduler-zookeeper:

condition:
service_healthy

volumes:

-
dolphinscheduler-logs:/opt/dolphinscheduler/logs

-
dolphinscheduler-shared-local:/opt/soft

networks:

-
dolphinscheduler


dolphinscheduler-worker:

image:
${HUB}/dolphinscheduler-worker:${TAG}

profiles:
 [
"all"
]

env_file:
.env

environment:

JAVA_OPTS:
-server
-Duser.timezone=${SPRING_JACKSON_TIME_ZONE}
-Xms8g
-Xmx8g
-Xmn4g
-XX:+PrintGCDetails
-Xloggc:gc.loo

g
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof

healthcheck:

test:
 [ 
"CMD"
, 
"curl"
, 
"http://localhost:1235/actuator/health"
 ]

interval:
30s

timeout:
5s

retries:
3

depends_on:

dolphinscheduler-zookeeper:

condition:
service_healthy

volumes:

-
dolphinscheduler-worker-data:/tmp/dolphinscheduler

-
dolphinscheduler-logs:/opt/dolphinscheduler/logs

-
dolphinscheduler-shared-local:/opt/soft

-
/dolphinscheduler:/dolphinscheduler

networks:

-
dolphinscheduler


networks:

dolphinscheduler:

driver:
bridge


volumes:

#注释掉postgresql volume

#dolphinscheduler-postgresql:

dolphinscheduler-mysql:

dolphinscheduler-zookeeper:

dolphinscheduler-worker-data:

dolphinscheduler-logs:

dolphinscheduler-shared-local:

修改.env

复制代码
# Docker Hub 镜像前缀和版本标签

HUB=apache

TAG=3.1.9


# MySQL 配置

MYSQL_ROOT_PASSWORD=root # 替换为您希望的 MySQL root 密码

MYSQL_DATABASE=dolphinscheduler # Dolphinscheduler 使用的数据库名称

MYSQL_USERNAME=root # Dolphinscheduler 连接 MySQL 的用户名 (通常就是 root)

MYSQL_PASSWORD=root # Dolphinscheduler 连接 MySQL 的密码 (与 root 密码相同)


# Dolphinscheduler 数据库连接配置 (通过 Docker Compose 传递给 DS 服务)

TZ=Asia/Shanghai

#指定使用MySQL数据库

DATABASE=mysql

SPRING_JACKSON_TIME_ZONE=GMT+8

SPRING_DATASOURCE_URL=jdbc:mysql://dolphinscheduler-mysql:3306/${MYSQL_DATABASE}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true

SPRING_DATASOURCE_USERNAME=${MYSQL_USERNAME}

SPRING_DATASOURCE_PASSWORD=${MYSQL_PASSWORD}

REGISTRY_ZOOKEEPER_CONNECT_STRING=dolphinscheduler-zookeeper:2181

MASTER_FETCH_COMMAND_NUM=10

初始化数据库

复制代码
docker compose --profile schema up -d

启动集群

复制代码
docker compose --profile all up -d

启动Worker服务

复制代码
docker compose up -d dolphinscheduler-worker

启动Master服务

复制代码
docker compose up -d dolphinscheduler-master

启动Alert服务

复制代码
docker compose up -d olphinscheduler-alert

启动api服务

复制代码
docker compose up -d dolphinscheduler-api

重启所有服务

复制代码
docker compose --profile all restart

作者:学通

来源:https://juejin.cn/post/7527944272277667855