flinkSql 将流和表的互相转换

流------>表

方式一

方式二

复制代码
方式一:写sql 
DataStreamSource<String> source = env.socketTextStream("localhost", 8881);
// 表名,流,字段名称
tableEnv.createTemporaryView("t_1",source,$("word"));

方式二:使用dsl
DataStreamSource<String> source = env.socketTextStream("localhost", 8881);
// 表名,流,字段名称
Table table = tableEnv.fromDataStream(source,$("word"));

表------>流

复制代码
Table table = tEnv.sqlQuery("select word,count(1) wordCount from t_1 group by word");

// 方式一:toAppendStream
DataStream<Row> appendStream = tEnv.toAppendStream(table, Row.class);

// 报错:toAppendStream doesn't support consuming update changes which is produced by node GroupAggregate(groupBy=[word], select=[word, SUM(num) AS sumNum])

// 这个不支持分组和聚合操作,若出现聚合操作使用方式二将表转为流

//方式二:toRetractStream
DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(table, Row.class);

wordCount案例

方式一:使用 sql

复制代码
package com.bigdata.day07;

import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import static org.apache.flink.table.api.Expressions.$;

/**
 * @基本功能:
 * @program:flinkProject
 * @author: 堇年
 * @create:2024-11-28 14:42:27
 **/
public class _06_flink_wordcounnt {

    public static void main(String[] args) throws Exception {

        //1. env-准备环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        // 获取tableEnv对象
        // 通过env 获取一个table 环境
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

        DataStreamSource<String> source = env.socketTextStream("localhost", 8881);
        SingleOutputStreamOperator<String> flatMap = source.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String value, Collector<String> out) throws Exception {
                String[] split = value.split(",");
                for (String s : split) {
                    out.collect(s);
                }
            }
        });
        //2. 创建表对象
        tEnv.createTemporaryView("t_1",flatMap,$("word"));
        //3. 编写sql语句
        Table table = tEnv.sqlQuery("select word,count(1) wordCount from t_1 group by word");
        //4. 将Table变为stream流
        //使用toAppendStream时会报错 因为有聚合操作
        //DataStream<Row> appendStream = tEnv.toAppendStream(table, Row.class);
        // toAppendStream doesn't support consuming update changes which is produced by node GroupAggregate(groupBy=[word], select=[word, SUM(num) AS sumNum])
        // 在这里可以映射为ROW对象,也可以映射为自己定义的实体类
        DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(table, Row.class);
        retractStream.filter(new FilterFunction<Tuple2<Boolean, Row>>() {
            @Override
            public boolean filter(Tuple2<Boolean, Row> value) throws Exception {
                return value.f0;
            }
        }).print();


        //5. execute-执行
        env.execute();
    }
}

方式二:使用 dsl 语句

复制代码
package com.bigdata.day07;

import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import static org.apache.flink.table.api.Expressions.$;

public class _06_flink_wordcounnt_dsl {

    public static void main(String[] args) throws Exception {

        //1. env-准备环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        // 获取tableEnv对象
        // 通过env 获取一个table 环境
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

        DataStreamSource<String> source = env.socketTextStream("localhost", 8881);
        SingleOutputStreamOperator<String> flatMap = source.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String value, Collector<String> out) throws Exception {
                String[] split = value.split(",");
                for (String s : split) {
                    out.collect(s);
                }
            }
        });
        //2. 创建表对象
        Table table = tEnv.fromDataStream(flatMap,$("word"));

        //3. 编写sql语句
        Table rsTable = table.groupBy($("word")).select($("word"),$("word").count().as("wordcount"));
        rsTable.printSchema();

        //4. 将Table变为stream流

        DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(rsTable, Row.class);
        retractStream.filter(new FilterFunction<Tuple2<Boolean, Row>>() {
            @Override
            public boolean filter(Tuple2<Boolean, Row> value) throws Exception {
                return value.f0;
            }
        }).print();


        //5. execute-执行
        env.execute();
    }
}

结果展示

复制代码
+I 表示有一条新数据进行了插入
+U 表示有一条已存在的数据有插入了一条,需要进行更新
-U 在+U前表示,先删除原本的,在update新的
相关推荐
liulilittle2 小时前
C++ TAP(基于任务的异步编程模式)
服务器·开发语言·网络·c++·分布式·任务·tap
码字的字节2 小时前
ZooKeeper在Hadoop中的协同应用:从NameNode选主到分布式锁实现
hadoop·分布式·zookeeper·分布式锁
im_AMBER3 小时前
学习日志19 python
python·学习
武子康4 小时前
Java-80 深入浅出 RPC Dubbo 动态服务降级:从雪崩防护到配置中心秒级生效
java·分布式·后端·spring·微服务·rpc·dubbo
_Kayo_7 小时前
VUE2 学习笔记6 vue数据监测原理
vue.js·笔记·学习
数据与人工智能律师7 小时前
数字迷雾中的安全锚点:解码匿名化与假名化的法律边界与商业价值
大数据·网络·人工智能·云计算·区块链
chenchihwen7 小时前
大模型应用班-第2课 DeepSeek使用与提示词工程课程重点 学习ollama 安装 用deepseek-r1:1.5b 分析PDF 内容
人工智能·学习
超浪的晨7 小时前
Java UDP 通信详解:从基础到实战,彻底掌握无连接网络编程
java·开发语言·后端·学习·个人开发
mykyle9 小时前
Elasticsearch-ik分析器
大数据·elasticsearch·jenkins
悠哉悠哉愿意9 小时前
【电赛学习笔记】MaxiCAM 项目实践——与单片机的串口通信
笔记·python·单片机·嵌入式硬件·学习·视觉检测