Flink-1.20集群部署

1、修改配置

1.1、zoo.cfg

# The number of milliseconds of each tick
tickTime=2000

# The number of ticks that the initial  synchronization phase can take
initLimit=10

# The number of ticks that can pass between  sending a request and getting an acknowledgement
syncLimit=5

# The directory where the snapshot is stored.
# dataDir=/tmp/zookeeper

# The port at which the clients will connect
clientPort=2181

# ZooKeeper quorum peers
server.1=localhost:2888:3888
# server.2=host:peer-port:leader-port

1.2、workers

localhost

1.3、masters

localhost

1.4、config.yaml

env:
  java:
    opts:
      all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED

#  # log4j 2 configuration
  log:
    level: TRACE
    max: 10
    dir: /data/flink/logs


#==============================================================================
# Common
#==============================================================================

jobmanager:
  bind-host: 0.0.0.0
  rpc:
    address: 0.0.0.0
    port: 6123
  memory:
    process:
      size: 2048m
  execution:
    failover-strategy: region
    # 归档已完成作业的路径
  archive:
    fs:
      dir: hdfs://hdfs-ha/flink/completed-jobs/

taskmanager:
  bind-host: 0.0.0.0
  host: 0.0.0.0
  # The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
  numberOfTaskSlots: 1
  memory:
    process:
      size: 2048m

parallelism:
  # The parallelism used for programs that did not specify and other parallelism.
  default: 1

fs:
  default-scheme: hdfs://hdfs-ha

#==============================================================================
# Fault tolerance and checkpointing
#==============================================================================

# # Execution checkpointing related parameters. Please refer to CheckpointConfig and CheckpointingOptions for more details.
execution:
  checkpointing:
    interval: 3min
    externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
    max-concurrent-checkpoints: 1
    min-pause: 1min
    mode: EXACTLY_ONCE
    timeout: 15min
    tolerable-failed-checkpoints: 2
    # 非对齐检查点
    unaligned: false
    incremental: true
    dir: hdfs://hdfs-ha/flink/checkpoints
    savepoint-dir: hdfs://hdfs-ha/flink/savepoints


state:
  backend:
    type: rocksdb

#==============================================================================
# Rest & web frontend
#==============================================================================

rest:
  # The address to which the REST client will connect to
  address: 0.0.0.0
  bind-address: 0.0.0.0
  # # The port to which the REST client connects to. If rest.bind-port has
  # # not been specified, then the server will bind to this port as well.
  # port: 8081
  # # Port range for the REST and web server to bind to.
  # bind-port: 8080-8090
  profiling:
    enable: true
web:
  submit:
    enable: true
  cancel:
    enable: true

#==============================================================================
# Advanced
#==============================================================================

# io:
#   tmp:
#     dirs: /tmp

# classloader:

classloader:
  resolve:
    order: child-first
  check-leaked-classloader: false

# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)

historyserver:
  web:
    address: 0.0.0.0
    port: 8082
  archive:
    fs:
      dir: hdfs://hdfs-ha/flink/completed-jobs/
    fs.refresh-interval: 10000

2、环境集成

Flink SQL、Paimon环境集成

3、测试案例

3.1、TopSpeedWindowing示例

./bin/flink run-application \
-t yarn-application \
-Dyarn.application.name=TopSpeedWindowing-Task \
-Djob.name=TopSpeedWindowing-Job \
./examples/streaming/TopSpeedWindowing.jar

3.2、WindowJoin示例

./bin/flink run-application -t yarn-application \
-Dyarn.application.name=Flink-WindowJoin-Demo \
./examples/streaming/WindowJoin.jar \
--windowSize 3000 \
--slideSize 3000 \
--rate 4