flink-start源码

jobSubmit

testCse

java 复制代码
final StreamExecutionEnvironment env = new StreamExecutionEnvironment(configuration);
//将算子添加进transformArrayList中
        env.fromCollection(Collections.singletonList(42))
            .addSink(new DiscardingSink<>());
        return env.execute();

StreamExecutionEnvironment

构建streamGraph(进行相关配置初始化,并未真正进行构建)并异步执行

java 复制代码
    public JobExecutionResult execute(String jobName) throws Exception {
        Preconditions.checkNotNull(jobName, "Streaming Job name should not be null.");
        final StreamGraph streamGraph = getStreamGraph();
        streamGraph.setJobName(jobName);
        return execute(streamGraph);
    }

public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    //异步执行streamGraph
     final JobClient jobClient = executeAsync(streamGraph);

            try {
                final JobExecutionResult jobExecutionResult;

                if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
                    jobExecutionResult = jobClient.getJobExecutionResult().get();
                } else {
                    jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
                }

                jobListeners.forEach(
                        jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));

                return jobExecutionResult;
            } catch (Throwable t) {
                ...
        }
    
    
    public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
            final PipelineExecutorFactory executorFactory =
                executorServiceLoader.getExecutorFactory(configuration);


        CompletableFuture<JobClient> jobClientFuture =
                executorFactory
            //获取执行器,具体实现类如下图
                        .getExecutor(configuration)
            //实例化执行器并执行
                        .execute(streamGraph, configuration, userClassloader);

        try {
            JobClient jobClient = jobClientFuture.get();
            //提交成功
            jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
            return jobClient;
        } catch (ExecutionException executionException) {
            final Throwable strippedException =
                    ExceptionUtils.stripExecutionException(executionException);
            jobListeners.forEach(
                //提交失败
                    jobListener -> jobListener.onJobSubmitted(null, strippedException));

            throw new FlinkException(
                    String.format("Failed to execute job '%s'.", streamGraph.getJobName()),
                    strippedException);
        }
    }

executor实现:

LocalExecutorFactory

java 复制代码
public class LocalExecutorFactory implements PipelineExecutorFactory {


@Override
public PipelineExecutor getExecutor(final Configuration configuration) {
    //创建执行器
    return LocalExecutor.create(configuration);
}
 ...   
}

LocalExecutor

java 复制代码
public static LocalExecutor create(Configuration configuration) {
//创建cluster
    return new LocalExecutor(configuration, MiniCluster::new);
}

LocalExecutor

java 复制代码
    @Override
    public CompletableFuture<JobClient> execute(
            Pipeline pipeline, Configuration configuration, ClassLoader userCodeClassloader)
            throws Exception {
        checkNotNull(pipeline);
        checkNotNull(configuration);

        Configuration effectiveConfig = new Configuration();
        effectiveConfig.addAll(this.configuration);
        effectiveConfig.addAll(configuration);

        // we only support attached execution with the local executor.
        checkState(configuration.getBoolean(DeploymentOptions.ATTACHED));

        final JobGraph jobGraph = getJobGraph(pipeline, effectiveConfig);

        return PerJobMiniClusterFactory.createWithFactory(effectiveConfig, miniClusterFactory)
            //提交作业
                .submitJob(jobGraph, userCodeClassloader);
    }

 /** Starts a {@link MiniCluster} and submits a job. */
    public CompletableFuture<JobClient> submitJob(
            JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
        MiniClusterConfiguration miniClusterConfig =
                getMiniClusterConfig(jobGraph.getMaximumParallelism());
        MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
        
        //启动cluster
        miniCluster.start();

        return miniCluster
                .submitJob(jobGraph)
                .thenApplyAsync(
                        FunctionUtils.uncheckedFunction(
                                submissionResult -> {
                                    org.apache.flink.client.ClientUtils
                                            .waitUntilJobInitializationFinished(
                                                    () ->
                                                            miniCluster
                                                                    .getJobStatus(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    () ->
                                                            miniCluster
                                                                    .requestJobResult(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    userCodeClassloader);
                                    return submissionResult;
                                }))
                .thenApply(
                        result ->
                                new MiniClusterJobClient(
                                        result.getJobID(),
                                        miniCluster,
                                        userCodeClassloader,
                                        MiniClusterJobClient.JobFinalizationBehavior
                                                .SHUTDOWN_CLUSTER))
                .whenComplete(
                        (ignored, throwable) -> {
                            if (throwable != null) {
                                // We failed to create the JobClient and must shutdown to ensure
                                // cleanup.
                                shutDownCluster(miniCluster);
                            }
                        })
                .thenApply(Function.identity());
    }

MiniCluster

java 复制代码
public CompletableFuture<JobSubmissionResult> submitJob(JobGraph jobGraph) {
    final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture =
            getDispatcherGatewayFuture();
    final CompletableFuture<InetSocketAddress> blobServerAddressFuture =
            createBlobServerAddress(dispatcherGatewayFuture);
    final CompletableFuture<Void> jarUploadFuture =
            uploadAndSetJobFiles(blobServerAddressFuture, jobGraph);
    final CompletableFuture<Acknowledge> acknowledgeCompletableFuture =
            jarUploadFuture
                    .thenCombine(
                            dispatcherGatewayFuture,
                            (Void ack, DispatcherGateway dispatcherGateway) ->
                                    dispatcherGateway.submitJob(jobGraph, rpcTimeout))
                    .thenCompose(Function.identity());
    return acknowledgeCompletableFuture.thenApply(
            (Acknowledge ignored) -> new JobSubmissionResult(jobGraph.getJobID()));
}
相关推荐
IT毕设梦工厂35 分钟前
大数据毕业设计选题推荐-基于大数据的客户购物订单数据分析与可视化系统-Hadoop-Spark-数据可视化-BigData
大数据·hadoop·数据分析·spark·毕业设计·源码·bigdata
java水泥工36 分钟前
基于Echarts+HTML5可视化数据大屏展示-白茶大数据溯源平台V2
大数据·echarts·html5
广州腾科助你拿下华为认证3 小时前
华为考试:HCIE数通考试难度分析
大数据·华为
在未来等你5 小时前
Elasticsearch面试精讲 Day 17:查询性能调优实践
大数据·分布式·elasticsearch·搜索引擎·面试
大数据CLUB8 小时前
基于spark的澳洲光伏发电站选址预测
大数据·hadoop·分布式·数据分析·spark·数据开发
ratbag6720138 小时前
当环保遇上大数据:生态环境大数据技术专业的课程侧重哪些领域?
大数据
计算机编程小央姐10 小时前
跟上大数据时代步伐:食物营养数据可视化分析系统技术前沿解析
大数据·hadoop·信息可视化·spark·django·课程设计·食物
智数研析社10 小时前
9120 部 TMDb 高分电影数据集 | 7 列全维度指标 (评分 / 热度 / 剧情)+API 权威源 | 电影趋势分析 / 推荐系统 / NLP 建模用
大数据·人工智能·python·深度学习·数据分析·数据集·数据清洗
潘达斯奈基~11 小时前
《大数据之路1》笔记2:数据模型
大数据·笔记
寻星探路11 小时前
数据库造神计划第六天---增删改查(CRUD)(2)
java·大数据·数据库