flink入门代码

flink入门代码

java 复制代码
package com.lyj.sx.flink.wordCount;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;


public class LocalWithWebUI {
    public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
         DataStreamSource<String> source = env.socketTextStream("pxj62", 8889);
         SingleOutputStreamOperator<Tuple2<String, Integer>> summed = source.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {
                for (String string : s.split(" ")) {
                    collector.collect(Tuple2.of(string, 1));
                }
            }
        }).keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            @Override
            public String getKey(Tuple2<String, Integer> s) throws Exception {
                return s.f0;
            }
        }).sum(1);
         summed.print();
         env.execute("pxj");
    }
}
java 复制代码
package com.lyj.sx.flink.wordCount;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class StreamingWordCount {
    public static void main(String[] args) throws  Exception{
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         int parallelism = env.getParallelism();
        System.out.println("parallelism:" + parallelism);
        DataStreamSource<String> source = env.socketTextStream("pxj62", 8881);

        System.out.println("source"+source.getParallelism());
         SingleOutputStreamOperator<Tuple2<String, Integer>> summed = source.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {
                String[] strings = s.split(" ");
                for (String string : strings) {
                    collector.collect(Tuple2.of(string, 1));
                }
            }
        }).keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            @Override
            public String getKey(Tuple2<String, Integer> s) throws Exception {
                return s.f0;
            }
        }).sum(1);
         summed.print();
         env.execute("pxj");
    }
}
java 复制代码
package com.lyj.sx.flink.wordCount;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;


public class StreamingWordCountV3 {
    public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         DataStreamSource<String> source = env.socketTextStream("pxj62", 8889);
         SingleOutputStreamOperator<Tuple2<String, Integer>> data = source.flatMap(new MyFlatMap());
         SingleOutputStreamOperator<Tuple2<String, Integer>> summed = data.keyBy(0).sum(1);
         summed.print();
         env.execute("pxj");
    }

    public static  class MyFlatMap implements FlatMapFunction<String, Tuple2<String,Integer>> {

        @Override
        public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {
            for (String string : s.split(" ")) {
                collector.collect(Tuple2.of(string,1));
            }
        }
    }
}
java 复制代码
package com.lyj.sx.flink.day02;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class ReadTextFileDemo {
    public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         DataStreamSource<String> source = env.readTextFile("data/a.txt");
         source.map(new MapFunction<String, Tuple2<String,Integer>>() {
             Tuple2<String,Integer> s1;

             @Override
             public Tuple2<String, Integer> map(String s) throws Exception {

                 String[] strings = s.split(" ");
                 for (String string : strings) {
                      s1=Tuple2.of(string,1);
                 }
                 return s1;
             }
         }).print();
         env.execute("pxj");

    }
}
java 复制代码
package com.lyj.sx.flink.day02;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.Arrays;
import java.util.List;
import java.util.UUID;

public class CustomNoParSource {
    public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
         System.out.println("环境执行的并行度:"+env.getParallelism());
         DataStreamSource<String> source = env.addSource(new Mysource2());
        System.out.println("source的并行度为:"+source.getParallelism());
        source.print();
//         env.execute("pxj");
        env.execute();

    }

    private static class Mysource1 implements SourceFunction<String> {
        //启动,并产生数据,产生的数据用SourceContext输出
        @Override
        public void run(SourceContext<String> cx) throws Exception {
             List<String> lists = Arrays.asList("a", "b", "c", "pxj", "sx", "lyj");
            for (String list : lists) {
               cx.collect(list);
            }

        }
        //将Source停掉
        @Override
        public void cancel() {

        }
    }

    private static class Mysource2 implements  SourceFunction<String>{
        private Boolean flag=true;
        @Override
        public void run(SourceContext<String> cx) throws Exception {
            System.out.println("run....");
            while (flag){
                cx.collect(UUID.randomUUID().toString());
            }

        }

        @Override
        public void cancel() {
            System.out.println("cancel");
            flag=false;
        }
    }
}

作者:pxj_sx(潘陈)

日期:2024-04-11 0:26:20

相关推荐
大大大大晴天6 小时前
每天认识一个组件:VictoriaLogs
大数据
学长毕设9 小时前
基于SpringBoot 博物馆文物租赁及修复管理系统-附源码
大数据·数据库·spring boot
玖米GEO运营王总监11 小时前
成都AI数字人视频怎么批量生产?流程与合规标注
大数据·人工智能·音视频
龙腾AI白云12 小时前
数字孪生与计算机视觉:让机器看懂现场
大数据·数据库·机器学习·知识图谱·pygame
starzy199015 小时前
Structured Streaming 案例剖析:一个 8 万 QPS 实时大屏的完整落地与三次线上事故复盘
大数据·spark
大大大大晴天1 天前
浅聊维度建模中的星型模型与雪花模型
大数据
数字新视界1 天前
能耗管理系统提升企业能源效率的实践与展望
大数据·人工智能·数据中心·机房管理·模块化机房
大大大大晴天1 天前
别让金额越算越多:事实表与维度表设计避坑指南
大数据
如夏倾城1 天前
入侵检测系统(IDS)原理、部署模式与 IPS/防火墙对比
大数据·网络·人工智能·网络安全·云原生·架构·kubernetes
wuyk5551 天前
Spider-第20讲-综合实战可上线爬虫
大数据·爬虫·python·elasticsearch