pyflink task并行度问题

复制代码
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from  pyflink.datastream.connectors import  DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime




logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)

# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types

env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
相关推荐
mortimer1 小时前
安装NVIDIA Parakeet时,我遇到的两个Pip“小插曲”
python·github
@昵称不存在1 小时前
Flask input 和datalist结合
后端·python·flask
赵英英俊2 小时前
Python day25
python
东林牧之2 小时前
Django+celery异步:拿来即用,可移植性高
后端·python·django
何双新2 小时前
基于Tornado的WebSocket实时聊天系统:从零到一构建与解析
python·websocket·tornado
AntBlack3 小时前
从小不学好 ,影刀 + ddddocr 实现图片验证码认证自动化
后端·python·计算机视觉
凪卄12133 小时前
图像预处理 二
人工智能·python·深度学习·计算机视觉·pycharm
巫婆理发2223 小时前
强化学习(第三课第三周)
python·机器学习·深度神经网络
seasonsyy4 小时前
1.安装anaconda详细步骤(含安装截图)
python·深度学习·环境配置
半新半旧4 小时前
python 整合使用 Redis
redis·python·bootstrap