flink: 向clickhouse写数据

一、依赖

复制代码
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>flink-proj</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-csv</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

<!--        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>-->

        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-api-java -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>1.11.1</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.bahir/flink-connector-redis -->
        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.12</artifactId>
            <version>1.1.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch7_2.12</artifactId>
            <version>1.11.1</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.22.1</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>2.0.12</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.30</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.12</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>ru.yandex.clickhouse</groupId>
            <artifactId>clickhouse-jdbc</artifactId>
            <version>0.3.2</version>
        </dependency>





    </dependencies>

</project>

二、clickhouse中建表

复制代码
create table userinfo(username varchar(100) primary key,passwd varchar(100));

三、通过Sink把从文件中读取的内容写到clickhouse

复制代码
package cn.edu.tju.demo;

import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.jdbc.JdbcStatementBuilder;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import java.sql.PreparedStatement;
import java.sql.SQLException;

public class Test16B {
    private static String CLICKHOUSE_SERVER = "xx.xx.xx.xx";
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();

        DataStream<String> mySource = environment.readTextFile("demo.txt");

        String sql = "insert into userinfo(username,passwd) values(?,?) ";
        JdbcConnectionOptions jdbcBuild = new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                .withDriverName("ru.yandex.clickhouse.ClickHouseDriver")
                .withUrl("jdbc:clickhouse://" + CLICKHOUSE_SERVER + ":8123/default")
                .withUsername("default")
                .withPassword("yourpassword")
                .build();

        mySource.addSink(JdbcSink.sink(sql, new JdbcStatementBuilder<String>() {
            @Override
            public void accept(PreparedStatement ps, String s) throws SQLException {
                ps.setString(1, s);
                ps.setString(2, s);

            }
        }, jdbcBuild));

        environment.execute("my job");

    }







}
相关推荐
TaoSense14 分钟前
Milvus向量数据库介绍
大数据·人工智能
智海观潮29 分钟前
聊聊Spark的分区
java·大数据·spark
洛克大航海33 分钟前
集群环境安装与部署 Hadoop
大数据·hadoop·ubuntu·集群部署 hadoop
EasyCVR2 小时前
赋能智慧水利:视频汇聚平台EasyCVR智慧水利工程视频管理系统解决方案
大数据
程序员洲洲2 小时前
使用亮数据爬虫API一键式爬取Facebook数据
大数据·数据·亮数据·bright data·爬虫api
汽车仪器仪表相关领域3 小时前
工业商业安全 “哨兵”:GT-NHVR-20-A1 点型可燃气体探测器实操解析与场景适配
大数据·人工智能·功能测试·安全·安全性测试
ctrigger3 小时前
电子信息三胞胎:电子信息工程、电子科学技术、电子信息科学技术
大数据·注册电气工程师
得物技术4 小时前
告别数据无序:得物数据研发与管理平台的破局之路
大数据·数据库·数据分析
武子康4 小时前
大数据-137 ClickHouse MergeTree 实战指南|分区、稀疏索引与合并机制 存储结构 一级索引 跳数索引
大数据·后端·nosql
TDengine (老段)5 小时前
TDengine 数学函数 SIGN 用户手册
大数据·数据库·sql·时序数据库·iot·tdengine·涛思数据