from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
呆呆敲代码的小Y14 小时前
10 分钟搞懂 cua:开源 AI 操作电脑基础设施,附 Python 沙箱与 Agent 上手代码工具分享14 小时前
带店托管必用爆单AI选品,一人公司轻松掌握l12586514 小时前
# RAG低延迟架构设计:从5秒到500ms的优化全链路杨丰玮41814 小时前
从零手写Java飞机躲障碍游戏|Swing绘图、鼠标跟随、计时器碰撞检测实战(五)frjc14 小时前
阿里云 ACK 环境 Arthas 在线调试指南Python私教14 小时前
如意智影:如何让同一个人物在多个镜头里保持身份一致高洁0114 小时前
智能体的“记忆”难题三十岁老牛再出发15 小时前
8月21日总结草莓熊Lotso15 小时前
【Linux网络加餐】手动部署:SSH 与 Web 服务实战 + 底层原理全解析weixin_4407305015 小时前
python+request实现接口-参数化小结