from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
充钱大佬6 小时前
Python测试基础教程初心丨哈士奇8 小时前
Python 四大基础容器|列表篇明理的信封9 小时前
AI 基础设施的“去 Python 化“:Rust 与 C# 的两条替代路径麻雀飞吧10 小时前
2026年AI量化开发,先跑通小流程再加复杂功能daphne odera�10 小时前
PyCharm 中 Codex 插件启动失败:unknown variant default 的解决方法nbu04william11 小时前
Deepseek-api省token的用法测试老哥11 小时前
Pytest自动化测试详解坚持学习前端日记11 小时前
国产化适配全流程适配英伟达本地开发源图客12 小时前
云途物流API开发-鉴权认证(Java)三川69813 小时前
Tkinter库的学习记录06-变量类别