flink入门

java斗罗2023-11-22 9:25

1.安装flink，启动flink

文档地址：Apache Flink 1.3-SNAPSHOT 中文文档: Apache Flink 中文文档

代码：GitHub - apache/flink: Apache Flink

  1. 打开端口 端口号， 启动jar

    切换到flink 目录bin下

    [root@localhost ~]# cd /home/flink/flink-1.14.4/bin/

    运行

    [root@localhost bin]# ./start-cluster.sh

    ###开启端口9000
    nc -l 9000

    运行jar

    ./bin/flink run /home/flink/flink-1.14.4/examples/streaming/SocketWindowWordCount.jar --port 9000

3.测试jar,输入字符

注：1. 部署启动遇到的jar缺失

注释：jar可以下载源码查看，方法如图所示，也可以根据错误信息搜索对应的包

附： mysql+mq+mybatis +spring 需要的包

2,.代码

java 复制代码 
package com.javaland.flink.mq;

import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.rabbitmq.RMQSource;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;

public class Mq2Flink {

    /**
     * 实时监控mq数据，插入到mysql数据库
     */
    public static void mq2mysql() throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        RMQConnectionConfig connectionConfig = new RMQConnectionConfig.Builder()
                .setHost("localhost")
                .setPort(5672)
                .setUserName("test")
                .setPassword("test")
                .setVirtualHost("/")
                .build();

        final DataStream<String> stream = env
                .addSource(new RMQSource<String>( connectionConfig, "task_queue", true, new SimpleStringSchema()))
                .setParallelism(1);
        stream.addSink(new SinkToMySQL());
        stream.print();
        env.execute("mq数据插入到mysql");
    }

    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRestartStrategy(RestartStrategies.noRestart());
        RMQConnectionConfig connectionConfig = new RMQConnectionConfig.Builder()
                .setHost("localhost")
                .setPort(5672)
                .setUserName("test")
                .setPassword("test")
                .setVirtualHost("/")
                .build();

        final DataStream<String> stream = env
                .addSource(new RMQSource<String>( connectionConfig, "task_queue", true, new SimpleStringSchema()))
                .setParallelism(1);
        stream.addSink(new SinkToMySQL());
        stream.print();
        env.execute("mq数据插入到mysql");
    }

}
java 复制代码 
package com.javaland.flink.mq;

import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import com.javaland.flink.mapper.MessageMapper;
import com.javaland.flink.po.MessagePO;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.ibatis.datasource.pooled.PooledDataSource;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import java.util.List;


public class SinkToMySQL extends RichSinkFunction<String> {


    static MybatisSqlSessionFactoryBean sqlSessionFactory;
    static SqlSessionFactory sessionFactory;
    static SqlSession sqlSession;
    static {
        sqlSessionFactory = new MybatisSqlSessionFactoryBean();
            // 配置多数据源
            PooledDataSource pooledDataSource = new PooledDataSource();
            pooledDataSource.setDriver("com.mysql.cj.jdbc.Driver");
            pooledDataSource.setUsername("root");
            pooledDataSource.setPassword("root");
            pooledDataSource.setUrl("jdbc:mysql://localhost:3306/test?serverTimezone=Asia/Shanghai");
        sqlSessionFactory.setDataSource(pooledDataSource);
        try {
            sqlSessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath*:mapper/*.xml"));
            sqlSessionFactory.setTypeAliasesPackage("com.javaland.flink.po");
            sessionFactory = sqlSessionFactory.getObject();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }



    /**
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        sqlSession = sessionFactory.openSession();
    }

    /**
     * @throws Exception
     */
    @Override
    public void close() throws Exception {
        super.close();

    }

    /**
     * @param value
     * @param context
     */
    @Override
    public void invoke(String value, Context context) {
         MessageMapper messageMapper = sqlSession.getMapper(MessageMapper.class);
        MessagePO messagePO=new MessagePO();
            messagePO.setUsername(value);
        messageMapper.insert(messagePO);
            List<MessagePO> all = messageMapper.selectList(null);
           if(all!=null){
               for (int i = all.size() - 1; i >= 0; i--) {
                   System.out.println("查询的message:"+all.get(i));
               }
           }

    }




}
XML 复制代码 
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>com.javaland</groupId>
        <artifactId>javaland</artifactId>
        <version>0.0.1</version>
    </parent>
    <packaging>jar</packaging>
    <groupId>org.javaland</groupId>
    <artifactId>javaland-flink</artifactId>

    <properties>
        <flink.version>1.14.4</flink.version>
    </properties>

    <dependencies>
 
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-jdbc_2.11</artifactId>
            <version>1.10.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-rabbitmq_2.12</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>com.rabbitmq</groupId>
            <artifactId>amqp-client</artifactId>
            <version>5.9.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-shaded-force-shading</artifactId>
            <version>14.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.11.1</version>
        </dependency>

        <dependency>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
            <version>1.2</version>
        </dependency>
 <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.31</version>
        </dependency>
 <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.5.3.2</version>
        </dependency>
         <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
                <version>1.2.18</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>2.4</version>
                <executions>
                    <execution>
                        <id>mq2flink</id>
                        <phase>package</phase>
                        <goals>
                            <goal>jar</goal>
                        </goals>
                        <configuration>
                            <classifier>Mq2Flink</classifier>
                            <archive>
                                <manifestEntries>
                                    <program-class>com.javaland.flink.Mq2Flink</program-class>
                                </manifestEntries>
                            </archive>
                        </configuration>
                    </execution>
                    <execution>
                        <id>Flink2Mq</id>
                        <phase>package</phase>
                        <goals>
                            <goal>jar</goal>
                        </goals>
                        <configuration>
                            <classifier>Flink2Mq</classifier>
                            <archive>
                                <manifestEntries>
                                    <program-class>com.javaland.flink.Flink2Mq</program-class>
                                </manifestEntries>
                            </archive>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>

</build>

</project>
  1. 最后打包的好处就是可以部署多个job
相关推荐
expect7g1 分钟前
Flink-Checkpoint-2.OperatorChain
后端·flink
天翼云开发者社区10 分钟前
Doris-HDFS LOAD常见问题汇总（二）
大数据·doris
简婷1870199877535 分钟前
源网荷储 + 零碳园区：一场关于能源与未来的双向奔赴
大数据·人工智能·能源
2201_756776771 小时前
网络安全初级
大数据·elasticsearch·搜索引擎
m0_703323671 小时前
SEO外包服务甄选指南：避开陷阱，精准匹配
大数据·人工智能
zzywxc7872 小时前
AI技术通过提示词工程（Prompt Engineering）正在深度重塑职场生态和行业格局，这种变革不仅体现在效率提升，更在重构人机协作模式。
java·大数据·开发语言·人工智能·spring·重构·prompt
张先shen2 小时前
Elasticsearch RESTful API入门：索引的增删改查完全指南
java·大数据·elasticsearch·搜索引擎·架构·全文检索·restful
yzx9910132 小时前
AI大模型平台
大数据·人工智能·深度学习·机器学习
189228048615 小时前
NW710NW713美光固态闪存NW719NW720
大数据·服务器·网络·人工智能·科技
诗旸的技术记录与分享6 小时前
Flink-1.19.0源码详解6-JobGraph生成-后篇
大数据·flink
热门推荐
01使用Ruby接入实时行情API教程02KGG转MP3工具|非KGM文件|解密音频03Coze扣子平台完整体验和实践（附国内和国际版对比）04深度神经网络训练过程与常见概念05集群聊天服务器---MySQL数据库的建立06扣子（coze）实战|我用扣子搭建了一个自动分析小红薯笔记内容的AI应用|详细步骤拆解07【无标题】08Java学习第十五部分——MyBatis09DeepSeek各版本说明与优缺点分析10LOT: 通过逻辑增强大型语言模型的零样本Chain-of-Thought推理能力