Mapreduce_Distinct数据去重

MapReduce中数据去重

输入如下的数据,统计其中的地址信息,并对输出的地址信息进行去重

实现方法:Map阶段输出的信息K2为想要去重的内容,利用Reduce阶段的聚合特点,对K2进行聚合,去重。在两阶段中,V2,V3,V4为Null

distinct.csv

bash 复制代码
John,30,New York
Jane,25,Los Angeles
Tom,33,New York
Doe,45,Chicago
Sam,26,Los Angeles
  1. main
bash 复制代码
package com.hadoop;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.streaming.io.InputWriter;



public class Main {
    public static void main(String[] args) throws Exception {
        Job job =Job.getInstance(new Configuration());
        job.setJarByClass(Main.class);

        job.setMapperClass(Map.class);
        job.setMapOutputKeyClass(Text.class);//k2
        job.setMapOutputValueClass(NullWritable.class);//v2

        job.setReducerClass(Reduce.class);
        job.setOutputKeyClass(Text.class);//k4
        job.setOutputValueClass(NullWritable.class);//v4


        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        job.waitForCompletion(true);

    }
}
  1. Map
bash 复制代码
package com.hadoop;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;
                                                    //Los Angeles
public class Map extends Mapper<LongWritable, Text,Text, NullWritable> {
   //Jane,25,Los Angeles
    @Override
    protected void map(LongWritable k1, Text v1,Context context)
            throws IOException, InterruptedException {
        String data =v1.toString();
        String words[]=data.split(",");

        context.write(new Text(words[2]),NullWritable.get());
    }
}
  1. Reduce
bash 复制代码
package com.hadoop;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class Reduce extends Reducer<Text, NullWritable,Text,NullWritable> {
    @Override
    protected void reduce(Text k3, Iterable<NullWritable> v3,Context context)
            throws IOException, InterruptedException {
        context.write(k3,NullWritable.get());
    }
}
  1. pom
bash 复制代码
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.hadoop</groupId>
    <artifactId>Mapreduce_Distinct</artifactId>
    <version>1.0-SNAPSHOT</version>

    <name>Mapreduce_Distinct</name>
    <description>wunaiieq</description>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <!--版本控制-->
        <hadoop.version>2.7.3</hadoop.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-api</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-streaming</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

    </dependencies>
    <!--构建配置-->
    <build>
        <plugins>
            <plugin>
                <!--声明-->
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.3.0</version>
                <!--具体配置-->
                <configuration>
                    <archive>
                        <manifest>
                            <!--jar包的执行入口-->
                            <mainClass>com.hadoop.Main</mainClass>
                        </manifest>
                    </archive>
                    <descriptorRefs>
                        <!--描述符,此处为预定义的,表示创建一个包含项目所有依赖的可执行 JAR 文件;
                        允许自定义生成jar文件内容-->
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <!--执行配置-->
                <executions>
                    <execution>
                        <!--执行配置ID,可修改-->
                        <id>make-assembly</id>
                        <!--执行的生命周期-->
                        <phase>package</phase>
                        <goals>
                            <!--执行的目标,single表示创建一个分发包-->
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>
  1. 效果
相关推荐
宅小海3 小时前
scala String
大数据·开发语言·scala
小白的白是白痴的白3 小时前
11.17 Scala练习:梦想清单管理
大数据
java1234_小锋3 小时前
Elasticsearch是如何实现Master选举的?
大数据·elasticsearch·搜索引擎
Java 第一深情7 小时前
零基础入门Flink,掌握基本使用方法
大数据·flink·实时计算
MXsoft6187 小时前
华为服务器(iBMC)硬件监控指标解读
大数据·运维·数据库
PersistJiao8 小时前
Spark 分布式计算中网络传输和序列化的关系(二)
大数据·网络·spark·序列化·分布式计算
九河云8 小时前
如何对AWS进行节省
大数据·云计算·aws
FreeIPCC9 小时前
谈一下开源生态对 AI人工智能大模型的促进作用
大数据·人工智能·机器人·开源
梦幻通灵9 小时前
ES分词环境实战
大数据·elasticsearch·搜索引擎
Elastic 中国社区官方博客9 小时前
Elasticsearch 中的热点以及如何使用 AutoOps 解决它们
大数据·运维·elasticsearch·搜索引擎·全文检索