flink on yarn 远程提交

java 复制代码
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.client.cli.CliFrontend;
import org.apache.flink.client.cli.CustomCommandLine;
import org.apache.flink.client.cli.DefaultCLI;
import org.apache.flink.client.cli.GenericCLI;
import org.apache.flink.client.deployment.ClusterDeploymentException;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.deployment.application.ApplicationConfiguration;
import org.apache.flink.client.program.ClusterClientProvider;
import org.apache.flink.configuration.*;
import org.apache.flink.runtime.security.SecurityConfiguration;
import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.yarn.YarnClientYarnClusterInformationRetriever;
import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.flink.yarn.YarnClusterInformationRetriever;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Test;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.UndeclaredThrowableException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;

import static org.apache.flink.util.Preconditions.checkNotNull;


@Slf4j
public class AppTestV1 {


    @Test
    public void submitJobWithYarnDesc() throws ClusterDeploymentException {
        // hadoop
        String hadoopConfDir = "C:\\Users\\HPN-21-117\\software\\configClusterhadoop\\configurations\\cdh5";
//        String hadoopConfDir = "C:\\Users\\HPN-21-117\\software\\configClusterhadoop\\configurations\\cdh6";
        //flink的本地配置目录,为了得到flink的配置
        String flinkConfDir = "C:\\Users\\HPN-21-117\\software\\flink-1.12.7\\conf";
        //存放flink集群相关的jar包目录
        String flinkLibs = "hdfs://cdh-node0.hypers.com:8020/flink/1.12.7/lib";
        //用户jar
        String userJarPath =  "hdfs://cdh-node0.hypers.com:8020/flink/demo/TopSpeedWindowing.jar";
        String flinkDistJar = "hdfs://cdh-node0.hypers.com:8020/flink/1.12.7/lib/flink-dist_2.12-1.12.7.jar";
//        String flinkDistJar = "hdfs://cdh-node0.hypers.com:8022/flink/lib";
        String[] args = "".split("\\s+");
        String appMainClass = "org.apache.flink.streaming.examples.windowing.TopSpeedWindowing";

        YarnClient yarnClient = YarnUtils.getYarnClient(hadoopConfDir);
        yarnClient.start();

        Configuration flinkConf = GlobalConfiguration.loadConfiguration(flinkConfDir);
        //set run model
        flinkConf.setString(DeploymentOptions.TARGET, YarnDeploymentTarget.APPLICATION.getName());
        //set application name
        flinkConf.setString(YarnConfigOptions.APPLICATION_NAME, "onYarnApiSubmitCase");
        //flink on yarn dependency
        flinkConf.set(YarnConfigOptions.PROVIDED_LIB_DIRS, Collections.singletonList(new Path(flinkLibs).toString()));
        flinkConf.set(YarnConfigOptions.FLINK_DIST_JAR, flinkDistJar);
        flinkConf.set(PipelineOptions.JARS, Collections.singletonList(new Path(userJarPath).toString()));
        //设置:资源/并发度
        flinkConf.setInteger(CoreOptions.DEFAULT_PARALLELISM, 1);
        flinkConf.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.parse("1G"));
        flinkConf.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.parse("1G"));
        flinkConf.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);


        ClusterSpecification clusterSpecification = new ClusterSpecification
                .ClusterSpecificationBuilder()
                .setMasterMemoryMB(1024)
                .setTaskManagerMemoryMB(1024)
                .setSlotsPerTaskManager(2)
                .createClusterSpecification();

        YarnClusterInformationRetriever ycir = YarnClientYarnClusterInformationRetriever.create(yarnClient);

        YarnConfiguration yarnConf = (YarnConfiguration) yarnClient.getConfig();

        ApplicationConfiguration appConfig = new ApplicationConfiguration(args, appMainClass);

        YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
                flinkConf,
                yarnConf,
                yarnClient,
                ycir,
                false);

        ClusterClientProvider<ApplicationId> applicationCluster =
                yarnClusterDescriptor.deployApplicationCluster( clusterSpecification, appConfig );

        yarnClient.stop();

    }

    @Test
    public  void submitJobWithCliForte() throws Exception {

        System.setProperty("ENV_FLINK_CONF_DIR", "C:\\Users\\HPN-21-117\\software\\flink-1.14.6\\conf");
        System.setProperty("FLINK_CONF_DIR", "C:\\Users\\HPN-21-117\\software\\flink-1.14.6\\conf");

        // 1. find the configuration directory
        final String configurationDirectory = getConfigurationDirectoryFromDir("C:\\Users\\HPN-21-117\\software\\flink-1.14.6\\conf");

        // 2. load the global configuration
        final Configuration configuration =
                GlobalConfiguration.loadConfiguration(configurationDirectory);

        // 3. load the custom command lines
        final List<CustomCommandLine> customCommandLines =
                loadCustomCommandLines(configuration, configurationDirectory);

        try {
            final CliFrontend cli = new CliFrontend(configuration, customCommandLines);

            SecurityUtils.install(new SecurityConfiguration(cli.getConfiguration()));
            String[] args = "run-application -t yarn-application hdfs://cdh-node0.hypers.com:8022/flink/demo/TopSpeedWindowing.jar".split("\\s+");
            int retCode =
                    SecurityUtils.getInstalledContext().runSecured(() -> cli.parseAndRun(args));
            System.exit(retCode);
        } catch (Throwable t) {
            final Throwable strippedThrowable =
                    ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
            log.error("Fatal error while running command line interface.", strippedThrowable);
            strippedThrowable.printStackTrace();
            System.exit(31);
        }

    }

    public static List<CustomCommandLine> loadCustomCommandLines(
            Configuration configuration, String configurationDirectory) {
        List<CustomCommandLine> customCommandLines = new ArrayList<>();
        customCommandLines.add(new GenericCLI(configuration, configurationDirectory));

        //	Command line interface of the YARN session, with a special initialization here
        //	to prefix all options with y/yarn.
        final String flinkYarnSessionCLI = "org.apache.flink.yarn.cli.FlinkYarnSessionCli";
        try {
            customCommandLines.add(
                    loadCustomCommandLine(
                            flinkYarnSessionCLI,
                            configuration,
                            configurationDirectory,
                            "y",
                            "yarn"));
        } catch (NoClassDefFoundError | Exception e) {
            final String errorYarnSessionCLI = "org.apache.flink.yarn.cli.FallbackYarnSessionCli";
            try {
                log.info("Loading FallbackYarnSessionCli");
                customCommandLines.add(loadCustomCommandLine(errorYarnSessionCLI, configuration));
            } catch (Exception exception) {
                log.warn("Could not load CLI class {}.", flinkYarnSessionCLI, e);
            }
        }

        //	Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get
        // the
        //	      active CustomCommandLine in order and DefaultCLI isActive always return true.
        customCommandLines.add(new DefaultCLI());

        return customCommandLines;
    }

    /**
     * Loads a class from the classpath that implements the CustomCommandLine interface.
     *
     * @param className The fully-qualified class name to load.
     * @param params The constructor parameters
     */
    private static CustomCommandLine loadCustomCommandLine(String className, Object... params)
            throws Exception {

        Class<? extends CustomCommandLine> customCliClass =
                Class.forName(className).asSubclass(CustomCommandLine.class);

        // construct class types from the parameters
        Class<?>[] types = new Class<?>[params.length];
        for (int i = 0; i < params.length; i++) {
            checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
            types[i] = params[i].getClass();
        }

        Constructor<? extends CustomCommandLine> constructor = customCliClass.getConstructor(types);

        return constructor.newInstance(params);
    }

    public static String getConfigurationDirectoryFromDir(String env_flink_conf_dir) {

//        String env_flink_conf_dir = System.getenv(flinkConfDir);

        String location = Optional.ofNullable(env_flink_conf_dir)
                .filter(dir -> new File(dir).exists())
                .orElseThrow(() -> new RuntimeException(
                        "The configuration directory '"
                                + env_flink_conf_dir
                                + "', specified in the '"
                                + ConfigConstants.ENV_FLINK_CONF_DIR
                                + "' environment variable, does not exist."));

        return location;
    }

    public static String getConfigurationDirectoryFromEnv() {
        System.setProperty("MY_VAR", "value");

        String env_flink_conf_dir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);

        String location = Optional.ofNullable(env_flink_conf_dir)
                .filter(dir -> new File(dir).exists())
                .orElseThrow(() -> new RuntimeException(
                        "The configuration directory '"
                                + env_flink_conf_dir
                                + "', specified in the '"
                                + ConfigConstants.ENV_FLINK_CONF_DIR
                                + "' environment variable, does not exist."));

        return location;
    }
}
相关推荐
数字新视界6 小时前
数据中心DCIM管理系统助力提升机房管理效率与安全性
大数据·人工智能·物联网·信息与通信·动环监控系统
阿里云大数据AI技术7 小时前
孩子王基于阿里云Data Agent的智能体实践:300+Agent 覆盖400+业务场景
大数据·人工智能·agent
武子康9 小时前
H3-World 扩展跳跃与开门:新动作怎样进入训练数据?
大数据·人工智能·算法
软件技术NINI10 小时前
DeepSeek 大模型落地应用实战指南
大数据·人工智能
wuyk55513 小时前
Spider-第11讲-工程化改造与可维护性
大数据·爬虫·elasticsearch·搜索引擎
墨林陌14 小时前
AI 热点日报(2026-11-16):黄仁勋或重返英伟达,华为发具身世界模型,ChatGPT Wallet打通支付
大数据
starzy199014 小时前
Structured Streaming 结构化流概述:把流数据当作一张无限增长的表
大数据·spark
wuyk55515 小时前
Spider-第12讲-asyncio异步并发爬取
大数据·爬虫·elasticsearch·搜索引擎
筑梦之人16 小时前
Elasticsearch 9.x 从入门到精通(4/7):聚合分析与文本分析
大数据·elasticsearch·jenkins
数字新视界1 天前
信创动环监控在机房环境管理中的智能化应用与实践分析
大数据·人工智能·物联网·信息与通信·动环监控系统