from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
知识中的海王2 小时前
js逆向入门图灵爬虫练习平台第六题碳基学AI3 小时前
北京大学DeepSeek内部研讨系列:AI在新媒体运营中的应用与挑战|122页PPT下载方法forestsea3 小时前
Python进阶编程总结袖清暮雨4 小时前
Python刷题笔记乌旭5 小时前
AI芯片混战:GPU vs TPU vs NPU的算力与能效博弈MinggeQingchun5 小时前
Python - 爬虫-网页抓取数据-库requests拓端研究室TRL6 小时前
Python贝叶斯回归、强化学习分析医疗健康数据拟合截断删失数据与参数估计3实例wolf犭良6 小时前
27、Python 数据库操作入门(SQLite)从基础到实战精讲sa100276 小时前
基于Python的网络爬虫技术研究