from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
import json
import re
import logging
import sys
from pyflink.datastream.state import ValueStateDescriptor, MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer, TypeInformation,FlinkKafkaProducer
from pyflink.common.typeinfo import Types
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
from datetime import datetime
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s-%(levelname)s-%(message)s")
logger = logging.getLogger(__name__)
# ���� StreamExecutionEnvironment ����
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
env.add_jars("file:///root/flink-sql-connector-kafka_2.11-1.14.4.jar")
from pyflink.datastream import DataStream, StreamExecutionEnvironment
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction, MapFunction
from pyflink.common.typeinfo import Types
env = StreamExecutionEnvironment.get_execution_environment()
data = DataStream(env._j_stream_execution_environment.socketTextStream('192.168.137.201', 8899))
#调用map算子,封装成一个task,并行度为8,有8个subtask
ds1=data.map(lambda s: s.upper()).set_parallelism(8)
##sink算子,并行度为4
ds1.print().set_parallelism(4)
pyflink task并行度问题
scan7242024-05-09 20:45
相关推荐
信誓旦旦的程序猿6 小时前
【Python量化实战宝典 #20】动量、反转择时手册:趋势跟随与均值回归(Python 可跑)好焊孙辉6 小时前
从六代螺柱焊机到AI质检:鸿栢科技的螺柱焊国产化路径benchmark_cc6 小时前
个人量化系统搭建:如何利用 QuantDash 实现 A 股日 K 线前复权数据的高效获取与质量校验?hhb_6186 小时前
Python异步队列实战:高效处理耗时任务2601_962885726 小时前
一个接口取 A 股+美股+港股:多市场数据方案对比SEO_juper6 小时前
Node.js 全站死链检测工具:从 sitemap 出发一次找全 404ljt27249606616 小时前
Python笔记(九)--asyncio老孟说禅7 小时前
自动化测试实战(二)2601_967987377 小时前
你们都用Python实现了哪些自动化办公?wflynn7 小时前
GitHub 今日推荐|Snapintosh:用 ARM EL2 监控器骗过内核,让骁龙芯片笔记本跑 macOS