flink-start源码

jobSubmit

testCse

java 复制代码
final StreamExecutionEnvironment env = new StreamExecutionEnvironment(configuration);
//将算子添加进transformArrayList中
        env.fromCollection(Collections.singletonList(42))
            .addSink(new DiscardingSink<>());
        return env.execute();

StreamExecutionEnvironment

构建streamGraph(进行相关配置初始化,并未真正进行构建)并异步执行

java 复制代码
    public JobExecutionResult execute(String jobName) throws Exception {
        Preconditions.checkNotNull(jobName, "Streaming Job name should not be null.");
        final StreamGraph streamGraph = getStreamGraph();
        streamGraph.setJobName(jobName);
        return execute(streamGraph);
    }

public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    //异步执行streamGraph
     final JobClient jobClient = executeAsync(streamGraph);

            try {
                final JobExecutionResult jobExecutionResult;

                if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
                    jobExecutionResult = jobClient.getJobExecutionResult().get();
                } else {
                    jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
                }

                jobListeners.forEach(
                        jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));

                return jobExecutionResult;
            } catch (Throwable t) {
                ...
        }
    
    
    public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
            final PipelineExecutorFactory executorFactory =
                executorServiceLoader.getExecutorFactory(configuration);


        CompletableFuture<JobClient> jobClientFuture =
                executorFactory
            //获取执行器,具体实现类如下图
                        .getExecutor(configuration)
            //实例化执行器并执行
                        .execute(streamGraph, configuration, userClassloader);

        try {
            JobClient jobClient = jobClientFuture.get();
            //提交成功
            jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
            return jobClient;
        } catch (ExecutionException executionException) {
            final Throwable strippedException =
                    ExceptionUtils.stripExecutionException(executionException);
            jobListeners.forEach(
                //提交失败
                    jobListener -> jobListener.onJobSubmitted(null, strippedException));

            throw new FlinkException(
                    String.format("Failed to execute job '%s'.", streamGraph.getJobName()),
                    strippedException);
        }
    }

executor实现:

LocalExecutorFactory

java 复制代码
public class LocalExecutorFactory implements PipelineExecutorFactory {


@Override
public PipelineExecutor getExecutor(final Configuration configuration) {
    //创建执行器
    return LocalExecutor.create(configuration);
}
 ...   
}

LocalExecutor

java 复制代码
public static LocalExecutor create(Configuration configuration) {
//创建cluster
    return new LocalExecutor(configuration, MiniCluster::new);
}

LocalExecutor

java 复制代码
    @Override
    public CompletableFuture<JobClient> execute(
            Pipeline pipeline, Configuration configuration, ClassLoader userCodeClassloader)
            throws Exception {
        checkNotNull(pipeline);
        checkNotNull(configuration);

        Configuration effectiveConfig = new Configuration();
        effectiveConfig.addAll(this.configuration);
        effectiveConfig.addAll(configuration);

        // we only support attached execution with the local executor.
        checkState(configuration.getBoolean(DeploymentOptions.ATTACHED));

        final JobGraph jobGraph = getJobGraph(pipeline, effectiveConfig);

        return PerJobMiniClusterFactory.createWithFactory(effectiveConfig, miniClusterFactory)
            //提交作业
                .submitJob(jobGraph, userCodeClassloader);
    }

 /** Starts a {@link MiniCluster} and submits a job. */
    public CompletableFuture<JobClient> submitJob(
            JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
        MiniClusterConfiguration miniClusterConfig =
                getMiniClusterConfig(jobGraph.getMaximumParallelism());
        MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
        
        //启动cluster
        miniCluster.start();

        return miniCluster
                .submitJob(jobGraph)
                .thenApplyAsync(
                        FunctionUtils.uncheckedFunction(
                                submissionResult -> {
                                    org.apache.flink.client.ClientUtils
                                            .waitUntilJobInitializationFinished(
                                                    () ->
                                                            miniCluster
                                                                    .getJobStatus(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    () ->
                                                            miniCluster
                                                                    .requestJobResult(
                                                                            submissionResult
                                                                                    .getJobID())
                                                                    .get(),
                                                    userCodeClassloader);
                                    return submissionResult;
                                }))
                .thenApply(
                        result ->
                                new MiniClusterJobClient(
                                        result.getJobID(),
                                        miniCluster,
                                        userCodeClassloader,
                                        MiniClusterJobClient.JobFinalizationBehavior
                                                .SHUTDOWN_CLUSTER))
                .whenComplete(
                        (ignored, throwable) -> {
                            if (throwable != null) {
                                // We failed to create the JobClient and must shutdown to ensure
                                // cleanup.
                                shutDownCluster(miniCluster);
                            }
                        })
                .thenApply(Function.identity());
    }

MiniCluster

java 复制代码
public CompletableFuture<JobSubmissionResult> submitJob(JobGraph jobGraph) {
    final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture =
            getDispatcherGatewayFuture();
    final CompletableFuture<InetSocketAddress> blobServerAddressFuture =
            createBlobServerAddress(dispatcherGatewayFuture);
    final CompletableFuture<Void> jarUploadFuture =
            uploadAndSetJobFiles(blobServerAddressFuture, jobGraph);
    final CompletableFuture<Acknowledge> acknowledgeCompletableFuture =
            jarUploadFuture
                    .thenCombine(
                            dispatcherGatewayFuture,
                            (Void ack, DispatcherGateway dispatcherGateway) ->
                                    dispatcherGateway.submitJob(jobGraph, rpcTimeout))
                    .thenCompose(Function.identity());
    return acknowledgeCompletableFuture.thenApply(
            (Acknowledge ignored) -> new JobSubmissionResult(jobGraph.getJobID()));
}
相关推荐
非极限码农3 小时前
Neo4j图数据库上手指南
大数据·数据库·数据分析·neo4j
莫叫石榴姐5 小时前
SQL百题斩:从入门到精通,一站式解锁数据世界
大数据·数据仓库·sql·面试·职场和发展
Hello.Reader5 小时前
Flink 状态后端(State Backends)实战原理、选型、配置与调优
大数据·flink
dundunmm8 小时前
【每天一个知识点】[特殊字符] 大数据的定义及单位
大数据
IT森林里的程序猿8 小时前
基于Hadoop的京东电商平台手机推荐系统的设计与实现
大数据·hadoop·智能手机
笨蛋少年派8 小时前
MapReduce简介
大数据·mapreduce
秃头菜狗9 小时前
十四、运行经典案例 wordcount
大数据·linux·hadoop
INFINI Labs9 小时前
Elasticsearch 备份:方案篇
大数据·elasticsearch·搜索引擎·gateway·snapshot·backup·ccr
Java战神9 小时前
Hadoop
大数据·hadoop·分布式
望获linux10 小时前
【实时Linux实战系列】实时系统的可观测性:Prometheus 与 Grafana 集成
大数据·linux·服务器·开发语言·网络·操作系统