flink: table api之自定义聚合函数

复制代码
package cn.edu.tju.demo3;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.*;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.descriptors.*;
import org.apache.flink.table.functions.AggregateFunction;
import org.apache.flink.table.functions.ScalarFunction;
import org.apache.flink.table.functions.TableFunction;
import org.apache.flink.types.Row;

public class Test50 {
    private static String HOST_NAME = "xx.xx.xx.xx";
    private static int PORT = 9999;
    private static String DELIMITER ="\n";


    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);



        DataStream<String> socketDataInfo =  env.socketTextStream(HOST_NAME, PORT, DELIMITER);
        SingleOutputStreamOperator<DataInfo> dataInfoStream = socketDataInfo.map(new MapFunction<String, DataInfo>() {
            @Override
            public DataInfo map(String value) throws Exception {

                String[] stringList = value.split(",");
                DataInfo dataInfo = new DataInfo(Long.parseLong(
                        stringList[0]), stringList[1], Double.parseDouble(stringList[2]));
                return dataInfo;
            }
        });

        Table dataTable = tableEnv.fromDataStream(dataInfoStream,"ts,info,val");
        tableEnv.registerFunction("myAggregateFunction", new MyAggregateFunction());
        Table resultTable = dataTable.select("ts,info,val")
                        .groupBy("info")
                                .aggregate("myAggregateFunction(val) as avgVal" )
                                        .select("info, avgVal");

        tableEnv.createTemporaryView("dataInfo", dataTable);

        Table resultTableSql = tableEnv.sqlQuery(
                "select info,myAggregateFunction(val) from dataInfo group by info"

        );


        tableEnv.toRetractStream(resultTable, Row.class).print();
        tableEnv.toRetractStream(resultTableSql, Row.class).print("sql");

        env.execute("my job");

    }

    public static class DataInfo{
        private long ts;
        private String info;
        private double val;

        public long getTs() {
            return ts;
        }

        public void setTs(long ts) {
            this.ts = ts;
        }

        public String getInfo() {
            return info;
        }

        public void setInfo(String info) {
            this.info = info;
        }

        public double getVal() {
            return val;
        }

        public void setVal(double val) {
            this.val = val;
        }

        @Override
        public String toString() {
            return "DataInfo{" +
                    "ts=" + ts +
                    ", info='" + info + '\'' +
                    ", val='" + val + '\'' +
                    '}';
        }

        public DataInfo(long ts, String info, double val) {
            this.ts = ts;
            this.info = info;
            this.val = val;
        }

        public DataInfo() {

        }
    }

    //自定义聚合函数,实现getResult和方法
    public static class MyAggregateFunction extends AggregateFunction<Double, Tuple2<Double, Integer>> {

        @Override
        public Double getValue(Tuple2<Double, Integer> accumulator) {
            return accumulator.f0/accumulator.f1;
        }

        @Override
        public Tuple2<Double, Integer> createAccumulator() {
            return new Tuple2(0.0, 0);
        }

        public void accumulate(Tuple2<Double, Integer> accumulator, double d){
            accumulator.f1 += 1;
            accumulator.f0 += d;
        }


    }
}

nc -lk 9999

输入:

复制代码
1689999831,ffff,34.2
1689999832,ffff,35.3

结果

相关推荐
万粉变现经纪人10 分钟前
如何解决pip安装报错ModuleNotFoundError: No module named ‘sympy’问题
python·beautifulsoup·pandas·scikit-learn·pyqt·pip·scipy
小朋友,你是否有很多问号?17 分钟前
Spark10- RDD转DataFrame的三种方式
大数据·javascript·spark
xiaohouzi11223333 分钟前
Python读取视频-硬解和软解
python·opencv·ffmpeg·视频编解码·gstreamer
念念不忘 必有回响37 分钟前
Pygame模块化实战:从零构建Aliens射击游戏全流程(一)
python·游戏·pygame
☼←安于亥时→❦8 小时前
PyTorch 梯度与微积分
人工智能·pytorch·python
青鱼入云8 小时前
【面试场景题】电商订单系统分库分表方案设计
大数据·面试·职场和发展
程序员三藏8 小时前
2025最新的软件测试面试八股文(800+道题)
自动化测试·软件测试·python·功能测试·测试工具·面试·职场和发展
Pocker_Spades_A8 小时前
Python快速入门专业版(二十三):for循环基础:遍历字符串、列表与range()函数(计数案例)
python
闲人编程9 小时前
图像去雾算法:从物理模型到深度学习实现
图像处理·人工智能·python·深度学习·算法·计算机视觉·去雾
在未来等你9 小时前
Kafka面试精讲 Day 12:副本同步与数据一致性
大数据·分布式·面试·kafka·消息队列