FlinkSQL开发步骤
Concepts & Common API | Apache Flink
添加依赖：
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_2.11</artifactId>
<version>1.13.6</version>
<scope>provided</scope>
</dependency>
DataStream -> 表
第一种方案：
第二种方案：
表->DataStream
查询
Table风格/DSL风格
SQL风格
进行一个简单的Demo演示：
java
package com.bigdata.day07;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
public class Demo06FlinkSQL {
public static void main(String[] args) throws Exception {
//1. env-准备环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
// 获取tableEnv对象
// 通过env 获取一个table 环境
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
//2. source-加载数据
DataStreamSource<String> streamSource = env.socketTextStream("localhost", 8899);
// 需要将DataStream变为 Table
tEnv.createTemporaryView("table1",streamSource);
//3. transformation-数据处理转换
Table table = tEnv.sqlQuery("select * from table1");
//table.printSchema(); 这个是打印表结构的，不是打印表数据的
//4. 要想打印表中的数据是没办法的，需要将table转换为DataStream
DataStream<Row> appendStream = tEnv.toAppendStream(table, Row.class);
appendStream.print();
//5. execute-执行
env.execute();
}
}
因为DataStream中是Row 类型，所以打印的格式是Row 这个类中的toString方法决定的。这个地方的 +I 的意思是新增的数据。
因为我们经常编写flinksql，所以创建一个模板：
java
#if (${PACKAGE_NAME} && ${PACKAGE_NAME} != "")package ${PACKAGE_NAME};#end
#parse("File Header.java")
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
public class ${NAME} {
public static void main(String[] args) throws Exception {
//1. env-准备环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
// 获取tableEnv对象
// 通过env 获取一个table 环境
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
//2. 创建表对象
//3. 编写sql语句
//4. 将Table变为stream流
//5. execute-执行
env.execute();
}
}根据这个可以做一个单词统计的案例：
第一版代码：
java
package com.bigdata.day08;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import static org.apache.flink.table.api.Expressions.$;
/**
*
* 输入的数据 hello,5
* hello,10 结果是 hello,15
**/
public class _01FlinkSQLWordCount {
public static void main(String[] args) throws Exception {
//1. env-准备环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
// 通过env 获取一个table 环境
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
//2. source-加载数据
DataStreamSource<String> dataStreamSource = env.socketTextStream("localhost", 6666);
// 将数据进行转换
DataStream<Tuple2<String, Integer>> mapStream = dataStreamSource.map(new MapFunction<String, Tuple2<String, Integer>>() {
@Override
public Tuple2<String, Integer> map(String s) throws Exception {
String[] arr = s.split(",");
String word = arr[0];
int num = Integer.valueOf(arr[1]);
return Tuple2.of(word, num);
}
});
// 一个单词是一列 默认是 f0
tEnv.createTemporaryView("table1",mapStream,$("word"),$("num"));
//3. 进行sql 查询
Table tableResult = tEnv.sqlQuery("select word,sum(num) as sumNum from table1 group by word");
//4. sink-数据输出
DataStream<Row> resultStream = tEnv.toAppendStream(tableResult, Row.class);
resultStream.print();
//5. execute-执行
env.execute();
}
}报错：
java
Exception in thread "main" org.apache.flink.table.api.TableException: toAppendStream doesn't support consuming update changes which is produced by node GroupAggregate(groupBy=[word], select=[word, SUM(num) AS sumNum])
at org.apache.flink.table.planner.plan.optimize.program.FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor.createNewNode(FlinkChangelogModeInferenceProgram.scala:395)
at org.apache.flink.table.planner.plan.optimize.program.FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor.visit(FlinkChangelogModeInferenceProgram.scala:186)
at org.apache.flink.table.planner.plan.optimize.program.FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor.org$apache$flink$table$planner$plan$optimize$program$FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor$$visitChild(FlinkChangelogModeInferenceProgram.scala:354)
at org.apache.flink.table.planner.plan.optimize.program.FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor$$anonfun$3.apply(FlinkChangelogModeInferenceProgram.scala:343)
at org.apache.flink.table.planner.plan.optimize.program.FlinkChangelogModeInferenceProgram$SatisfyModifyKindSetTraitVisitor$$anonfun$3.apply(FlinkChangelogModeInferenceProgram.scala:342)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.Range.foreach(Range.scala:160)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)解决方案：
//DataStream<Row> resultStream = tEnv.toAppendStream(tableResult, Row.class);
修改为：
DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(tableResult, Row.class);
第二版代码：
java
package com.bigdata.day08;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import static org.apache.flink.table.api.Expressions.$;
/**
* 输入的数据 hello,5
* hello,10 结果是 hello,15
**/
public class _01FlinkSQLWordCount {
public static void main(String[] args) throws Exception {
//1. env-准备环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
// 通过env 获取一个table 环境
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
//2. source-加载数据
DataStreamSource<String> dataStreamSource = env.socketTextStream("localhost", 6666);
// 将数据进行转换
DataStream<Tuple2<String, Integer>> mapStream = dataStreamSource.map(new MapFunction<String, Tuple2<String, Integer>>() {
@Override
public Tuple2<String, Integer> map(String s) throws Exception {
String[] arr = s.split(",");
String word = arr[0];
int num = Integer.valueOf(arr[1]);
return Tuple2.of(word, num);
}
});
// 一个单词是一列 默认是 f0
tEnv.createTemporaryView("table1",mapStream,$("word"),$("num"));
//3. 进行sql 查询
Table tableResult = tEnv.sqlQuery("select word,sum(num) as sumNum from table1 group by word");
//4. sink-数据输出
// 这个toAppendStream ，是新增的操作，不能用于分组
//DataStream<Row> resultStream = tEnv.toAppendStream(tableResult, Row.class);
// toRetractStream 使用这个带有缩进功能的方法，可以运行group by 等sql
DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(tableResult, Row.class);
retractStream.print();
//5. execute-执行
env.execute();
}
}toAppendStream: 适用于生成新的计算结果，并不会对老的计算结果进行修改。使用这个SQL语句中是不能出现分组的。
toRetractStream : 适用于对已经计算的结果进行更新，如果是true 代表新增，更新 false 代表遗弃
- I 表示新增
-U 更新前
+U 更新后
FlinkSQL-API：
**需求:**使用SQL和Table(DSL)两种方式对DataStream中的单词进行统计
java
package com.bigdata.sql;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class WC{
private String word;
private int num;
}
java
package com.bigdata.sql;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;
import static org.apache.flink.table.api.Expressions.$;
public class _02_WordCountDemo02 {
public static void main(String[] args) throws Exception {
//1. env-准备环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
// 获取tableEnv对象
// 通过env 获取一个table 环境
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
//2. 创建表对象
DataStreamSource<WC> dataStreamSource = env.fromElements(
new WC("hello",5),
new WC("hello",10),
new WC("world",1),
new WC("world",2)
);
// 第一种：使用纯sql的写法
tEnv.createTemporaryView("wordcount",dataStreamSource,$("word"),$("num"));
//3. 编写sql语句
Table table = tEnv.sqlQuery("select word,sum(num) wdcount from wordcount group by word ");
//4. 将Table变为stream流
// toAppendStream 不支持分组聚合操作
//DataStream<Row> appendStream = tEnv.toAppendStream(table, Row.class);
DataStream<Tuple2<Boolean, Row>> retractStream = tEnv.toRetractStream(table, Row.class);
//retractStream.print();
// 以上这些是SQL的写法，它还支持DSL的写法
Table table2 = tEnv.fromDataStream(dataStreamSource,$("word"),$("num"));
//Table resultTable = table2.groupBy($("word")).select($("word"), $("num").sum().as("wdcount"));
Table resultTable = table2.groupBy($("word")).select($("word"), $("num").sum()).as("word","num");
resultTable.printSchema();
DataStream<Tuple2<Boolean, Row>> resultStream= tEnv.toRetractStream(resultTable, Row.class);
resultStream.print();
resultStream.map(new MapFunction<Tuple2<Boolean, Row>, String>() {
@Override
public String map(Tuple2<Boolean, Row> value) throws Exception {
Row row = value.f1;
return row.getField("word")+","+row.getField("num");
}
}).print();
DataStream<Tuple2<Boolean, WC>> wcStream= tEnv.toRetractStream(resultTable, WC.class);
wcStream.print();
//5. execute-执行
env.execute();
}
}
java
Exception in thread "main" org.apache.flink.table.api.ValidationException: Too many fields referenced from an atomic type.
at org.apache.flink.table.typeutils.FieldInfoUtils.extractFieldInfoFromAtomicType(FieldInfoUtils.java:473)
假如遇到以上错误，请检查wc实体类中，是否有无参构造方法。