flink: 从kafka读取数据

一、添加相关依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>flink-proj</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.12</artifactId>
            <version>1.11.1</version>
        </dependency>

    </dependencies>

</project>

二、主类:

package cn.edu.tju.demo;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Arrays;
import java.util.List;
import java.util.Properties;

public class Test03 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment
                .getExecutionEnvironment();
        Properties properties  = new Properties();
        properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"xx.xx.xx.xx:9092");
        properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"testGroup");

        DataStreamSource<String> myTopic = environment.addSource(
                new FlinkKafkaConsumer<String>("myTopic", new SimpleStringSchema(), properties));
        myTopic.print();


        environment.execute("my job");

    }


}

kafka为2.x版本

相关推荐
Elastic 中国社区官方博客6 小时前
使用真实 Elasticsearch 进行高级集成测试
大数据·数据库·elasticsearch·搜索引擎·全文检索·jenkins·集成测试
好记性+烂笔头6 小时前
4 Spark Streaming
大数据·ajax·spark
圣心8 小时前
Kafka 使用说明(kafka官方文档中文)
分布式·kafka·linq
圣心8 小时前
Kafka中文文档
分布式·kafka
Leven1995278 小时前
Flink (十三) :Table API 与 DataStream API 的转换 (一)
数据库·sql·flink
大秦王多鱼9 小时前
Kafka SASL/PLAIN介绍
分布式·安全·kafka
苏苏大大9 小时前
Kafka下载
分布式·kafka
好记性+烂笔头10 小时前
3 Flink 运行架构
大数据·架构·flink
字节侠10 小时前
Flink2支持提交StreamGraph到Flink集群
大数据·flink·streamgraph·flink2·jobgraph
好记性+烂笔头13 小时前
4 Hadoop 面试真题
大数据·hadoop·面试