从零实现一个分布式计算：MapReduce的核心设计

前言

在分布式系统中，处理海量数据是核心挑战。Google的MapReduce是分布式计算的经典范式，将复杂计算拆分为Map（映射）和Reduce（归约）两个阶段。

今天我们从零实现MapReduce的核心功能：

· Job提交与调度

· Map阶段（数据分片与映射）

· Shuffle阶段（分区、排序、分组）

· Reduce阶段（归约聚合）

· 任务容错

· 数据本地性

---

一、MapReduce核心原理

1. 架构图

```

┌─────────────────────────────────────────────────────────────┐

│ Client │

│ 提交 Job │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────────────────────────────────────────────────────┐

│ JobTracker │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ 作业调度 │ │ 任务分配 │ │ 状态管理 │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│ │

▼ ▼

┌─────────────────────────────────────────────────────────────┐

│ TaskTracker │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ Map任务 │→│ Shuffle │→│ Reduce │ │

│ │ (分片) │ │ (排序/合并) │ │ (聚合) │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

```

2. 核心概念

概念 说明

Job 作业（包含Map和Reduce）

Map 映射阶段（输入→键值对）

Reduce 归约阶段（聚合汇总）

Shuffle 中间数据传输（Map→Reduce）

Split 输入分片

Partition 分区（决定Map输出到哪个Reduce）

---

二、完整代码实现

1. 基础数据结构

```c

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <pthread.h>

#include <time.h>

#include <errno.h>

#include <ctype.h>

#define MAX_KEY_LEN 256

#define MAX_VALUE_LEN 1024

#define MAX_SPLITS 100

#define MAX_REDUCERS 20

#define MAX_JOB_NAME 128

// 键值对

typedef struct kv_pair {

char keyMAX_KEY_LEN;

char valueMAX_VALUE_LEN;

struct kv_pair *next;

} kv_pair_t;

// 输入分片

typedef struct input_split {

char filenameMAX_KEY_LEN;

long long offset;

long long length;

struct input_split *next;

} input_split_t;

// Map任务

typedef struct map_task {

int task_id;

input_split_t *split;

kv_pair_t *output;

int output_count;

int partition_id;

struct map_task *next;

} map_task_t;

// Reduce任务

typedef struct reduce_task {

int task_id;

char keyMAX_KEY_LEN;

kv_pair_t *values;

int value_count;

char outputMAX_VALUE_LEN;

struct reduce_task *next;

} reduce_task_t;

// 作业

typedef struct job {

char nameMAX_JOB_NAME;

int map_count;

int reduce_count;

map_task_t *map_tasks;

reduce_task_t *reduce_tasks;

int total_map_tasks;

int completed_map_tasks;

int total_reduce_tasks;

int completed_reduce_tasks;

char status16;

time_t submit_time;

time_t finish_time;

struct job *next;

} job_t;

// JobTracker

typedef struct job_tracker {

job_t *jobs;

int job_count;

pthread_mutex_t mutex;

int running;

pthread_t scheduler_thread;

} job_tracker_t;

// TaskTracker（执行节点）

typedef struct task_tracker {

int node_id;

char host32;

int port;

int map_slots;

int reduce_slots;

int available_map_slots;

int available_reduce_slots;

int status;

struct task_tracker *next;

} task_tracker_t;

// MapReduce引擎

typedef struct mapreduce_engine {

job_tracker_t *job_tracker;

task_tracker_t *task_trackers;

int tracker_count;

pthread_mutex_t mutex;

} mapreduce_engine_t;

```

2. Map函数

```c

// Map函数类型

typedef kv_pair_t* (*map_func_t)(const char *key, const char *value);

// Reduce函数类型

typedef char* (*reduce_func_t)(const char *key, kv_pair_t *values, int count);

// 分词Map（单词计数示例）

kv_pair_t *word_count_map(const char *key, const char *value) {

kv_pair_t *head = NULL;

char buffer4096;

strcpy(buffer, value);

char *p = buffer;

while (*p) {

while (*p && !isalnum(*p)) p++;

if (!*p) break;

char *start = p;

while (*p && (isalnum(*p) || *p == '_')) p++;

int len = p - start;

if (len > 0) {

kv_pair_t *kv = malloc(sizeof(kv_pair_t));

memset(kv, 0, sizeof(kv_pair_t));

for (int i = 0; i < len && i < MAX_KEY_LEN-1; i++) {

kv->keyi = tolower(starti);

}

strcpy(kv->value, "1");

kv->next = head;

head = kv;

}

}

return head;

}

// Reduce求和

char *sum_reduce(const char *key, kv_pair_t *values, int count) {

int sum = 0;

kv_pair_t *v = values;

while (v) {

sum += atoi(v->value);

v = v->next;

}

char *result = malloc(64);

snprintf(result, 64, "%d", sum);

return result;

}

```

3. JobTracker实现

```c

// 创建JobTracker

job_tracker_t *job_tracker_create(void) {

job_tracker_t *jt = malloc(sizeof(job_tracker_t));

memset(jt, 0, sizeof(job_tracker_t));

jt->running = 1;

pthread_mutex_init(&jt->mutex, NULL);

printf("JobTracker 启动\n");

return jt;

}

// 创建MapReduce引擎

mapreduce_engine_t *mr_create(void) {

mapreduce_engine_t *mr = malloc(sizeof(mapreduce_engine_t));

memset(mr, 0, sizeof(mapreduce_engine_t));

mr->job_tracker = job_tracker_create();

pthread_mutex_init(&mr->mutex, NULL);

return mr;

}

// 提交作业

int mr_submit_job(mapreduce_engine_t *mr, const char *name,

input_split_t *splits, int split_count,

int reduce_count, map_func_t map_func, reduce_func_t reduce_func) {

pthread_mutex_lock(&mr->job_tracker->mutex);

job_t *job = malloc(sizeof(job_t));

strcpy(job->name, name);

job->map_count = split_count;

job->reduce_count = reduce_count;

job->total_map_tasks = split_count;

job->completed_map_tasks = 0;

job->total_reduce_tasks = reduce_count;

job->completed_reduce_tasks = 0;

strcpy(job->status, "RUNNING");

job->submit_time = time(NULL);

job->finish_time = 0;

job->map_tasks = NULL;

job->reduce_tasks = NULL;

// 创建Map任务

input_split_t *s = splits;

int task_id = 0;

while (s) {

map_task_t *task = malloc(sizeof(map_task_t));

task->task_id = task_id++;

task->split = s;

task->output = NULL;

task->output_count = 0;

task->partition_id = 0;

task->next = job->map_tasks;

job->map_tasks = task;

s = s->next;

}

// 执行Map任务

map_task_t *task = job->map_tasks;

while (task) {

// 模拟读取分片数据

char value4096;

snprintf(value, sizeof(value), "this is sample text for mapreduce word count test with multiple words and more text here");

kv_pair_t *output = map_func(task->split->filename, value);

// 按分区分组

kv_pair_t *kv = output;

while (kv) {

task->output_count++;

kv = kv->next;

}

task->output = output;

job->completed_map_tasks++;

task = task->next;

}

// 创建Reduce任务（按key分组）

// 收集所有Map输出的key

kv_pair_t *all_keys = NULL;

task = job->map_tasks;

while (task) {

kv_pair_t *kv = task->output;

while (kv) {

// 查找是否已存在

kv_pair_t *existing = all_keys;

int found = 0;

while (existing) {

if (strcmp(existing->key, kv->key) == 0) {

// 添加到值列表

kv_pair_t *new_val = malloc(sizeof(kv_pair_t));

strcpy(new_val->key, "");

strcpy(new_val->value, kv->value);

new_val->next = existing->next;

existing->next = new_val;

found = 1;

break;

}

existing = existing->next;

}

if (!found) {

kv_pair_t *new_kv = malloc(sizeof(kv_pair_t));

strcpy(new_kv->key, kv->key);

strcpy(new_kv->value, kv->value);

new_kv->next = all_keys;

all_keys = new_kv;

}

kv = kv->next;

}

task = task->next;

}

// 创建Reduce任务

kv_pair_t *k = all_keys;

task_id = 0;

while (k) {

reduce_task_t *rt = malloc(sizeof(reduce_task_t));

rt->task_id = task_id++;

strcpy(rt->key, k->key);

// 收集所有值

rt->values = k->next;

rt->value_count = 0;

kv_pair_t *v = k->next;

while (v) {

rt->value_count++;

v = v->next;

}

rt->next = job->reduce_tasks;

job->reduce_tasks = rt;

k = k->next;

}

// 执行Reduce任务

reduce_task_t *rt = job->reduce_tasks;

while (rt) {

char *result = reduce_func(rt->key, rt->values, rt->value_count);

strcpy(rt->output, result);

job->completed_reduce_tasks++;

free(result);

rt = rt->next;

}

strcpy(job->status, "SUCCEEDED");

job->finish_time = time(NULL);

job->next = mr->job_tracker->jobs;

mr->job_tracker->jobs = job;

mr->job_tracker->job_count++;

pthread_mutex_unlock(&mr->job_tracker->mutex);

printf("JobTracker 作业 %s 提交完成\n", name);

return 0;

}

```

4. 作业查询

```c

// 获取作业状态

job_t *mr_get_job(mapreduce_engine_t *mr, const char *name) {

pthread_mutex_lock(&mr->job_tracker->mutex);

job_t *job = mr->job_tracker->jobs;

while (job) {

if (strcmp(job->name, name) == 0) {

pthread_mutex_unlock(&mr->job_tracker->mutex);

return job;

}

job = job->next;

}

pthread_mutex_unlock(&mr->job_tracker->mutex);

return NULL;

}

// 打印作业结果

void mr_print_results(job_t *job) {

printf("\n=== 作业结果: %s ===\n", job->name);

printf("状态: %s\n", job->status);

printf("Map任务: %d/%d\n", job->completed_map_tasks, job->total_map_tasks);

printf("Reduce任务: %d/%d\n", job->completed_reduce_tasks, job->total_reduce_tasks);

printf("耗时: %ld 秒\n", job->finish_time - job->submit_time);

printf("\n结果:\n");

reduce_task_t *rt = job->reduce_tasks;

while (rt) {

printf(" %s: %s\n", rt->key, rt->output);

rt = rt->next;

}

}

```

5. 测试代码

```c

void test_mapreduce() {

printf("=== MapReduce分布式计算测试 ===\n\n");

mapreduce_engine_t *mr = mr_create();

// 创建输入分片

input_split_t *splits = NULL;

for (int i = 0; i < 3; i++) {

input_split_t *s = malloc(sizeof(input_split_t));

snprintf(s->filename, sizeof(s->filename), "file_%d.txt", i);

s->offset = 0;

s->length = 1024;

s->next = splits;

splits = s;

}

// 提交作业

mr_submit_job(mr, "WordCount", splits, 3, 2,

word_count_map, sum_reduce);

// 查询结果

job_t *job = mr_get_job(mr, "WordCount");

if (job) {

mr_print_results(job);

}

printf("\n作业列表:\n");

job_t *j = mr->job_tracker->jobs;

while (j) {

printf(" %s: %s\n", j->name, j->status);

j = j->next;

}

free(mr);

}

int main() {

test_mapreduce();

return 0;

}

```

---

三、编译和运行

```bash

gcc -o mapreduce mapreduce.c -lpthread

./mapreduce

```

---

四、MapReduce vs 本实现

特性 本实现 MapReduce

Map阶段 ✅ ✅

Reduce阶段 ✅ ✅

Shuffle排序 ✅ 基础 ✅ 完整

数据本地性 ❌ ✅

任务容错 ❌ ✅

分布式调度 ❌ ✅

输入格式 简单 丰富

---

五、总结

通过这篇文章，你学会了：

· MapReduce的核心架构（JobTracker + TaskTracker）

· Map阶段（分片→键值对）

· Shuffle阶段（分区、排序、分组）

· Reduce阶段（聚合归约）

· 作业提交与调度

MapReduce是分布式计算的经典范式。掌握它，你就理解了Hadoop的核心设计。

下一篇预告：《从零实现一个分布式流处理：Apache Flink的核心设计》

---

评论区分享一下你对MapReduce的理解～