flink: 通过Sink将数据写入MySQL

一、依赖添加

复制代码
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.30</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.12</artifactId>
            <version>1.11.1</version>
        </dependency>

二、两种方式写MySQL:

复制代码
package cn.edu.tju.demo;

import org.apache.flink.api.common.functions.*;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;
import org.apache.http.HttpHost;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;

import java.util.*;

public class Test16 {
    private static String MYSQL_SERVER = "139.198.0.216";
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();


        DataStream<String> mySource = environment.readTextFile("demo.txt");

        String sql = "insert into userinfo(username,passwd) values  " +
                "(?,?)";
        JdbcConnectionOptions jdbcBuild = new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                .withDriverName("com.mysql.jdbc.Driver")
                .withUrl("jdbc:mysql://xx.xx.xx.xx:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false")
                .withUsername("root")
                .withPassword("123456")
                .build();

        mySource.addSink(JdbcSink.sink(sql, (ps, str) -> {
            ps.setString(1, str);
            ps.setString(2, str);

        }, jdbcBuild));



        environment.execute("my job");

    }




}






package cn.edu.tju.demo;

import org.apache.flink.api.common.functions.*;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;
import org.apache.http.HttpHost;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.*;

public class Test17 {
    private static String MYSQL_SERVER = "xx.xx.xx.xx";
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();


        DataStream<String> mySource = environment.readTextFile("demo.txt");



        mySource.addSink(new MyJdbcSinkFunction());
        environment.execute("mysql");

    }


    public static class MyJdbcSinkFunction extends RichSinkFunction<String>{
        private Connection connection = null;
        private PreparedStatement insert = null;
        private PreparedStatement update = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            connection = DriverManager.getConnection("jdbc:mysql://xx.xx.xx.xx:3306/test" +
                    "?useUnicode=true&characterEncoding=utf-8&useSSL=false", "root", "123456");

            insert = connection.prepareStatement("insert into userinfo(username,passwd) values(?,?)");
            System.out.println("open mysql");
        }

        @Override
        public void close() throws Exception {
            connection.close();
        }

        @Override
        public void invoke(String value, Context context) throws Exception {

            insert.setString(1, value);
            insert.setString(2, value);
            insert.execute();
        }
    }


}
相关推荐
jack xu123 分钟前
高频面试题:如何保证数据库和es数据一致性
java·大数据·数据库·mysql·elasticsearch
炫彩@之星1 小时前
Mysql之UDF提权
mysql·udf提权
爱的叹息2 小时前
DeepSeek 大模型 + LlamaIndex + MySQL 数据库 + 知识文档 实现简单 RAG 系统
数据库·人工智能·mysql·langchain
橘猫云计算机设计2 小时前
net+MySQL中小民营企业安全生产管理系统(源码+lw+部署文档+讲解),源码可白嫖!
数据库·后端·爬虫·python·mysql·django·毕业设计
黄嚯嚯2 小时前
Mysql8.0 推出的强大功能 窗口函数(Window Functions)
数据库·mysql
神仙别闹2 小时前
基于Java+MySQL 实现(Web)日程管理系统
java·前端·mysql
苹果酱05673 小时前
2020-06-23 暑期学习日更计划(机器学习入门之路(资源汇总)+概率论)
java·vue.js·spring boot·mysql·课程设计
Lucky GGBond3 小时前
MySQL 报错解析:SQLSyntaxErrorException caused by extra comma before FROM
数据库·mysql
Java水解3 小时前
Mysql之存储过程
后端·mysql
漫步向前3 小时前
mysql主备延迟
mysql