flink-start源码

jobSubmit

testCse

java 复制代码
final StreamExecutionEnvironment env = new StreamExecutionEnvironment(configuration);
//将算子添加进transformArrayList中
        env.fromCollection(Collections.singletonList(42))
            .addSink(new DiscardingSink<>());
        return env.execute();

StreamExecutionEnvironment

构建streamGraph(进行相关配置初始化,并未真正进行构建)并异步执行

java 复制代码
    public JobExecutionResult execute(String jobName) throws Exception {
        Preconditions.checkNotNull(jobName, "Streaming Job name should not be null.");
        final StreamGraph streamGraph = getStreamGraph();
        streamGraph.setJobName(jobName);
        return execute(streamGraph);
    }

public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    //异步执行streamGraph
     final JobClient jobClient = executeAsync(streamGraph);

            try {
                final JobExecutionResult jobExecutionResult;

                if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
                    jobExecutionResult = jobClient.getJobExecutionResult().get();
                } else {
                    jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
                }

                jobListeners.forEach(
                        jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));

                return jobExecutionResult;
            } catch (Throwable t) {
                ...
        }
    
    
    public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
            final PipelineExecutorFactory executorFactory =
                executorServiceLoader.getExecutorFactory(configuration);


        CompletableFuture<JobClient> jobClientFuture =
                executorFactory
            //获取执行器,具体实现类如下图
                        .getExecutor(configuration)
            //实例化执行器并执行
                        .execute(streamGraph, configuration, userClassloader);

        try {
            JobClient jobClient = jobClientFuture.get();
            //提交成功
            jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
            return jobClient;
        } catch (ExecutionException executionException) {
            final Throwable strippedException =
                    ExceptionUtils.stripExecutionException(executionException);
            jobListeners.forEach(
                //提交失败
                    jobListener -> jobListener.onJobSubmitted(null, strippedException));

            throw new FlinkException(
                    String.format("Failed to execute job '%s'.", streamGraph.getJobName()),
                    strippedException);
        }
    }

executor实现:

LocalExecutorFactory

java 复制代码
public class LocalExecutorFactory implements PipelineExecutorFactory {


@Override
public PipelineExecutor getExecutor(final Configuration configuration) {
    //创建执行器
    return LocalExecutor.create(configuration);
}
 ...   
}

LocalExecutor

java 复制代码
public static LocalExecutor create(Configuration configuration) {
//创建cluster
    return new LocalExecutor(configuration, MiniCluster::new);
}

LocalExecutor

java 复制代码
    @Override
    public CompletableFuture<JobClient> execute(
            Pipeline pipeline, Configuration configuration, ClassLoader userCodeClassloader)
            throws Exception {
        checkNotNull(pipeline);
        checkNotNull(configuration);

        Configuration effectiveConfig = new Configuration();
        effectiveConfig.addAll(this.configuration);
        effectiveConfig.addAll(configuration);

        // we only support attached execution with the local executor.
        checkState(configuration.getBoolean(DeploymentOptions.ATTACHED));

        final JobGraph jobGraph = getJobGraph(pipeline, effectiveConfig);

        return PerJobMiniClusterFactory.createWithFactory(effectiveConfig, miniClusterFactory)
            //提交作业
                .submitJob(jobGraph, userCodeClassloader);
    }

 /** Starts a {@link MiniCluster} and submits a job. */
    public CompletableFuture<JobClient> submitJob(
            JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
        MiniClusterConfiguration miniClusterConfig =
                getMiniClusterConfig(jobGraph.getMaximumParallelism());
        MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
        
        //启动cluster
        miniCluster.start();

        return miniCluster
                .submitJob(jobGraph)
                .thenApplyAsync(
                        FunctionUtils.uncheckedFunction(
                                submissionResult -> {
                                    org.apache.flink.client.ClientUtils
                                            .waitUntilJobInitializationFinished(
                                                    () ->
                                                            miniCluster
                                                                    .getJobStatus(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    () ->
                                                            miniCluster
                                                                    .requestJobResult(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    userCodeClassloader);
                                    return submissionResult;
                                }))
                .thenApply(
                        result ->
                                new MiniClusterJobClient(
                                        result.getJobID(),
                                        miniCluster,
                                        userCodeClassloader,
                                        MiniClusterJobClient.JobFinalizationBehavior
                                                .SHUTDOWN_CLUSTER))
                .whenComplete(
                        (ignored, throwable) -> {
                            if (throwable != null) {
                                // We failed to create the JobClient and must shutdown to ensure
                                // cleanup.
                                shutDownCluster(miniCluster);
                            }
                        })
                .thenApply(Function.identity());
    }

MiniCluster

java 复制代码
public CompletableFuture<JobSubmissionResult> submitJob(JobGraph jobGraph) {
    final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture =
            getDispatcherGatewayFuture();
    final CompletableFuture<InetSocketAddress> blobServerAddressFuture =
            createBlobServerAddress(dispatcherGatewayFuture);
    final CompletableFuture<Void> jarUploadFuture =
            uploadAndSetJobFiles(blobServerAddressFuture, jobGraph);
    final CompletableFuture<Acknowledge> acknowledgeCompletableFuture =
            jarUploadFuture
                    .thenCombine(
                            dispatcherGatewayFuture,
                            (Void ack, DispatcherGateway dispatcherGateway) ->
                                    dispatcherGateway.submitJob(jobGraph, rpcTimeout))
                    .thenCompose(Function.identity());
    return acknowledgeCompletableFuture.thenApply(
            (Acknowledge ignored) -> new JobSubmissionResult(jobGraph.getJobID()));
}
相关推荐
B站_计算机毕业设计之家2 小时前
计算机毕业设计:Python农业数据可视化分析系统 气象数据 农业生产 粮食数据 播种数据 爬虫 Django框架 天气数据 降水量(源码+文档)✅
大数据·爬虫·python·机器学习·信息可视化·课程设计·农业
Apache Flink4 小时前
Flink Agents 0.1.0 发布公告
大数据·flink
潘达斯奈基~5 小时前
在使用spark的applyInPandas方法过程中,遇到类型冲突问题如何解决
大数据·笔记
火星资讯6 小时前
腾多多数字零售模式:从成本转嫁到全生态共赢的破局实践
大数据
望获linux7 小时前
【实时Linux实战系列】实时 Linux 的自动化基准测试框架
java·大数据·linux·运维·网络·elasticsearch·搜索引擎
金宗汉8 小时前
《宇宙递归拓扑学:基于自指性与拓扑流形的无限逼近模型》
大数据·人工智能·笔记·算法·观察者模式
直有两条腿8 小时前
【数据迁移】HBase Bulkload批量加载原理
大数据·数据库·hbase
Joy T8 小时前
海南蓝碳:生态财富与科技驱动的新未来
大数据·人工智能·红树林·海南省·生态区建设
风清再凯8 小时前
01-ELK安装ES,ES-head
大数据·elk·elasticsearch
Guheyunyi9 小时前
风险感知中枢:监测预警系统的架构与核心
大数据·运维·安全·重构·架构·自动化