from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
笨鸟先飞,勤能补拙4 小时前
AI 赋能网络安全:技术全景、成熟度评估与实战案例长和信泰光伏储能5 小时前
京津冀光伏发电:绿色能源的未来之路浦信仿真大讲堂6 小时前
从重复操作到自动化闭环:如何让 CST 与 Python 真正协同起来Gu Gu Study6 小时前
ScoutLoop开放域深度研究引擎(agent的初步设计想法)卷无止境7 小时前
写代码这件事,到底该讲究点什么?卷无止境7 小时前
循环复杂度到底在算什么,Python 代码怎么才能写得让人一看就懂lpfasd1237 小时前
MediaCrawler 项目深度分析Dxy12393102168 小时前
Python项目打包成EXE完整教程(PyInstaller实战避坑)bamb008 小时前
一个项目带你入门AI应用开发010566468 小时前
Python康复训练——常用标准库