flink: 通过Sink将数据写入MySQL

一、依赖添加

复制代码
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.30</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.12</artifactId>
            <version>1.11.1</version>
        </dependency>

二、两种方式写MySQL:

复制代码
package cn.edu.tju.demo;

import org.apache.flink.api.common.functions.*;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;
import org.apache.http.HttpHost;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;

import java.util.*;

public class Test16 {
    private static String MYSQL_SERVER = "139.198.0.216";
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();


        DataStream<String> mySource = environment.readTextFile("demo.txt");

        String sql = "insert into userinfo(username,passwd) values  " +
                "(?,?)";
        JdbcConnectionOptions jdbcBuild = new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                .withDriverName("com.mysql.jdbc.Driver")
                .withUrl("jdbc:mysql://xx.xx.xx.xx:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false")
                .withUsername("root")
                .withPassword("123456")
                .build();

        mySource.addSink(JdbcSink.sink(sql, (ps, str) -> {
            ps.setString(1, str);
            ps.setString(2, str);

        }, jdbcBuild));



        environment.execute("my job");

    }




}






package cn.edu.tju.demo;

import org.apache.flink.api.common.functions.*;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;
import org.apache.http.HttpHost;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.*;

public class Test17 {
    private static String MYSQL_SERVER = "xx.xx.xx.xx";
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();


        DataStream<String> mySource = environment.readTextFile("demo.txt");



        mySource.addSink(new MyJdbcSinkFunction());
        environment.execute("mysql");

    }


    public static class MyJdbcSinkFunction extends RichSinkFunction<String>{
        private Connection connection = null;
        private PreparedStatement insert = null;
        private PreparedStatement update = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            connection = DriverManager.getConnection("jdbc:mysql://xx.xx.xx.xx:3306/test" +
                    "?useUnicode=true&characterEncoding=utf-8&useSSL=false", "root", "123456");

            insert = connection.prepareStatement("insert into userinfo(username,passwd) values(?,?)");
            System.out.println("open mysql");
        }

        @Override
        public void close() throws Exception {
            connection.close();
        }

        @Override
        public void invoke(String value, Context context) throws Exception {

            insert.setString(1, value);
            insert.setString(2, value);
            insert.execute();
        }
    }


}
相关推荐
dleei1 小时前
MySql安装及SQL语句
数据库·后端·mysql
信徒_2 小时前
Mysql 在什么样的情况下会产生死锁?
android·数据库·mysql
苹果酱05673 小时前
Golang标准库——runtime
java·vue.js·spring boot·mysql·课程设计
·薯条大王9 小时前
MySQL联合查询
数据库·mysql
IT成长日记15 小时前
【MySQL基础】 JSON函数入门
mysql·json·json函数
天狼122216 小时前
第1章-3 MySQL的逻辑架构
mysql·逻辑架构
码觉客18 小时前
MySQL 5.7+ for Windows 解压缩版配置安装说明
mysql
THRUSTER1111119 小时前
MySQL-- 函数(单行函数):数值函数, 字符串函数
数据库·mysql·函数·navicat·单行函数
橙序研工坊19 小时前
MySQL的进阶语法7(索引-B+Tree 、Hash、聚集索引 、二级索引(回表查询)、索引的使用及设计原则
数据库·sql·mysql
宝哥大数据19 小时前
Flink内存模型--flink1.19.1
大数据·flink