from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
qq_413502021 小时前
如何创建CDB公共用户_C##前缀强制规则与CONTAINER=ALLyexuhgu2 小时前
CSS如何利用-checked实现纯CSS手风琴折叠_通过状态选择器控制区域高度AC赳赳老秦2 小时前
接口测试自动化:用 OpenClaw 对接 Postman,实现批量回归测试、测试报告自动生成与推送PILIPALAPENG2 小时前
第4周 Day 1:智能体记忆系统——给 Agent 一个"大脑"DavidTaozhe2 小时前
一文搞懂外汇接口怎么实时更新美元汇率用户78937733908532 小时前
Docker 部署踩坑记录:从“构建失败”到“服务跑通”,以及为什么数据被清空了再玩一会儿看代码2 小时前
如何理解神经网络中的权重参数?从一张图看懂模型参数量计算2301_779622412 小时前
mysql如何通过主从备份实现读写分离_配置mysql架构模式m0_741173332 小时前
HTML5中WebSocket在弱网环境下的延迟抖动算法补偿l1t2 小时前
astral-sh发布的musl和gnu版本standalone python 性能比较