从零实现一个分布式任务调度：Apache Airflow的核心设计

前言

在数据工程中，ETL任务、数据管道往往有复杂的依赖关系------A任务完成后才能执行B，B完成后才能执行C。Apache Airflow 是编排复杂任务依赖的事实标准。

今天我们从零实现Airflow的核心功能：

· DAG定义（有向无环图）

· 任务（Operator）

· 依赖关系

· 调度器（Scheduler）

· 执行器（Executor）

· 任务状态管理

· 任务重试与失败处理

---

一、Airflow核心原理

1. 架构图

```

┌─────────────────────────────────────────────────────────────┐

│ DAG定义 │

│ (Python代码) │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────────────────────────────────────────────────────┐

│ Scheduler │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ 解析DAG │ │ 调度任务 │ │ 触发执行 │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────────────────────────────────────────────────────┐

│ Executor │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ Local │ │ Celery │ │ Kubernetes │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────────────────────────────────────────────────────┐

│ Metadata DB │

│ (任务状态/运行历史) │

└─────────────────────────────────────────────────────────────┘

```

2. 核心概念

概念 说明

DAG 有向无环图（任务依赖图）

Operator 任务（执行单元）

Task Operator的实例

DAG Run DAG的一次执行

Task Instance Task的一次执行

Schedule 调度间隔（Cron表达式）

---

二、完整代码实现

1. 基础数据结构

```c

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <pthread.h>

#include <time.h>

#include <errno.h>

#include <math.h>

#define MAX_TASK_NAME 64

#define MAX_DAG_NAME 64

#define MAX_OPERATOR_ARGS 16

#define MAX_RETRIES 5

// 任务状态

typedef enum {

TASK_NONE = 0,

TASK_SCHEDULED,

TASK_RUNNING,

TASK_SUCCESS,

TASK_FAILED,

TASK_UPSTREAM_FAILED,

TASK_SKIPPED,

TASK_UP_FOR_RETRY

} task_state_t;

// 任务（Operator）

typedef struct task {

char task_idMAX_TASK_NAME;

int (*execute)(struct task *self, void **output);

char *argsMAX_OPERATOR_ARGS;

int arg_count;

struct task **upstream_tasks;

int upstream_count;

struct task **downstream_tasks;

int downstream_count;

int retries;

int max_retries;

int retry_delay_sec;

task_state_t state;

time_t start_time;

time_t end_time;

char output1024;

struct task *next;

} task_t;

// DAG

typedef struct dag {

char dag_idMAX_DAG_NAME;

task_t *tasks;

int task_count;

char schedule_interval64;

time_t start_date;

int catchup;

struct dag *next;

} dag_t;

// DAG Run

typedef struct dag_run {

char dag_idMAX_DAG_NAME;

char run_id64;

time_t execution_date;

char state16;

time_t start_time;

time_t end_time;

struct dag_run *next;

} dag_run_t;

// Airflow

typedef struct airflow {

dag_t *dags;

dag_run_t *dag_runs;

int dag_count;

pthread_mutex_t mutex;

int running;

int max_threads;

pthread_t scheduler_thread;

pthread_t executor_pool10;

} airflow_t;

```

2. DAG定义

```c

// 创建Airflow

airflow_t *airflow_create(int max_threads) {

airflow_t *af = malloc(sizeof(airflow_t));

memset(af, 0, sizeof(airflow_t));

af->max_threads = max_threads;

af->running = 1;

af->dag_count = 0;

pthread_mutex_init(&af->mutex, NULL);

printf("Airflow 启动，最大线程: %d\n", max_threads);

return af;

}

// 创建DAG

dag_t *airflow_create_dag(airflow_t *af, const char *dag_id,

const char *schedule_interval, time_t start_date) {

pthread_mutex_lock(&af->mutex);

dag_t *dag = malloc(sizeof(dag_t));

strcpy(dag->dag_id, dag_id);

strcpy(dag->schedule_interval, schedule_interval);

dag->start_date = start_date;

dag->catchup = 0;

dag->tasks = NULL;

dag->task_count = 0;

dag->next = af->dags;

af->dags = dag;

af->dag_count++;

pthread_mutex_unlock(&af->mutex);

printf("Airflow 创建DAG: %s\n", dag_id);

return dag;

}

// 创建任务

task_t *dag_create_task(dag_t *dag, const char *task_id,

int (*execute)(task_t*, void**)) {

task_t *task = malloc(sizeof(task_t));

strcpy(task->task_id, task_id);

task->execute = execute;

task->arg_count = 0;

task->upstream_tasks = NULL;

task->upstream_count = 0;

task->downstream_tasks = NULL;

task->downstream_count = 0;

task->retries = 0;

task->max_retries = 3;

task->retry_delay_sec = 60;

task->state = TASK_NONE;

task->start_time = 0;

task->end_time = 0;

task->next = dag->tasks;

dag->tasks = task;

dag->task_count++;

return task;

}

// 设置任务依赖: task1 >> task2 (task1完成后执行task2)

void dag_set_dependency(task_t *upstream, task_t *downstream) {

// 添加到上游任务列表

upstream->downstream_tasks = realloc(upstream->downstream_tasks,

sizeof(task_t*) * (upstream->downstream_count + 1));

upstream->downstream_tasksupstream-\>downstream_count++ = downstream;

// 添加到下游任务列表

downstream->upstream_tasks = realloc(downstream->upstream_tasks,

sizeof(task_t*) * (downstream->upstream_count + 1));

downstream->upstream_tasksdownstream-\>upstream_count++ = upstream;

}

// 检查任务是否可执行（所有上游任务已完成）

int task_is_ready(task_t *task) {

for (int i = 0; i < task->upstream_count; i++) {

if (task->upstream_tasksi->state != TASK_SUCCESS) {

return 0;

}

}

return 1;

}

```

3. 示例任务

```c

// 示例任务：打印消息

int print_task(task_t *task, void **output) {

printf("Task %s 执行中\n", task->task_id);

char *msg = task->args0;

if (msg) {

printf("Task %s: %s\n", task->task_id, msg);

snprintf(task->output, sizeof(task->output), "OK: %s", msg);

} else {

strcpy(task->output, "OK");

}

// 模拟执行时间

usleep(100000);

return 0;

}

// 示例任务：模拟失败

int fail_task(task_t *task, void **output) {

printf("Task %s 执行失败 (模拟)\n", task->task_id);

return -1;

}

// 示例任务：数据处理

int process_data_task(task_t *task, void **output) {

printf("Task %s 处理数据\n", task->task_id);

int input = 0;

if (task->args0) {

input = atoi(task->args0);

}

int result = input * 2 + 10;

snprintf(task->output, sizeof(task->output), "%d", result);

printf("Task %s: 输入=%d, 输出=%d\n", task->task_id, input, result);

usleep(50000);

return 0;

}

```

4. 调度器

```c

// 执行单个任务

void execute_task(airflow_t *af, task_t *task) {

pthread_mutex_lock(&af->mutex);

task->state = TASK_RUNNING;

task->start_time = time(NULL);

pthread_mutex_unlock(&af->mutex);

printf("Executor 执行任务: %s\n", task->task_id);

void *output = NULL;

int ret = task->execute(task, &output);

pthread_mutex_lock(&af->mutex);

task->end_time = time(NULL);

if (ret == 0) {

task->state = TASK_SUCCESS;

if (output) {

if (strlen(task->output) == 0) {

strcpy(task->output, (char*)output);

}

}

printf("Executor 任务 %s ✅ 成功\n", task->task_id);

} else if (task->retries < task->max_retries) {

task->state = TASK_UP_FOR_RETRY;

task->retries++;

printf("Executor 任务 %s 失败，重试 %d/%d\n",

task->task_id, task->retries, task->max_retries);

// 重试调度

task->state = TASK_SCHEDULED;

} else {

task->state = TASK_FAILED;

printf("Executor 任务 %s ❌ 失败 (已达最大重试)\n", task->task_id);

}

pthread_mutex_unlock(&af->mutex);

}

// 调度器主循环

void *scheduler_loop(void *arg) {

airflow_t *af = (airflow_t*)arg;

while (af->running) {

pthread_mutex_lock(&af->mutex);

// 遍历所有DAG

dag_t *dag = af->dags;

while (dag) {

// 检查调度时间

// 简化：所有任务都调度

task_t *task = dag->tasks;

while (task) {

if (task->state == TASK_NONE || task->state == TASK_SCHEDULED) {

if (task_is_ready(task)) {

task->state = TASK_SCHEDULED;

// 异步执行

pthread_t tid;

pthread_create(&tid, NULL, (void*(*)(void*))execute_task,

(void*)af);

pthread_detach(tid);

}

}

task = task->next;

}

dag = dag->next;

}

pthread_mutex_unlock(&af->mutex);

sleep(1);

}

return NULL;

}

// 启动Airflow

void airflow_start(airflow_t *af) {

pthread_create(&af->scheduler_thread, NULL, scheduler_loop, af);

printf("Airflow 调度器已启动\n");

}

// 停止Airflow

void airflow_stop(airflow_t *af) {

af->running = 0;

pthread_join(af->scheduler_thread, NULL);

}

```

5. 任务状态查询

```c

// 打印DAG状态

void airflow_print_dag_status(airflow_t *af, const char *dag_id) {

pthread_mutex_lock(&af->mutex);

dag_t *dag = af->dags;

while (dag) {

if (strcmp(dag->dag_id, dag_id) == 0) {

printf("\n=== DAG: %s ===\n", dag->dag_id);

task_t *task = dag->tasks;

while (task) {

const char *state_str\[\] = {

"NONE", "SCHEDULED", "RUNNING",

"SUCCESS", "FAILED", "UPSTREAM_FAILED",

"SKIPPED", "UP_FOR_RETRY"

};

printf(" %s: %s", task->task_id, state_strtask-\>state);

if (task->state == TASK_SUCCESS && task->output0) {

printf(" (输出: %s)", task->output);

}

printf("\n");

task = task->next;

}

break;

}

dag = dag->next;

}

pthread_mutex_unlock(&af->mutex);

}

```

6. 测试代码

```c

void test_airflow() {

printf("=== Airflow任务调度测试 ===\n\n");

airflow_t *af = airflow_create(4);

// 创建DAG

dag_t *dag = airflow_create_dag(af, "etl_pipeline", "0 0 * * *", time(NULL));

// 创建任务

task_t *task1 = dag_create_task(dag, "extract_data", print_task);

task_t *task2 = dag_create_task(dag, "transform_data", process_data_task);

task_t *task3 = dag_create_task(dag, "load_data", print_task);

task_t *task4 = dag_create_task(dag, "send_report", print_task);

task_t *task5 = dag_create_task(dag, "cleanup", print_task);

// 设置参数

task1->args0 = "Extracting data from source...";

task2->args0 = "10";

task3->args0 = "Loading processed data to warehouse...";

task4->args0 = "Sending daily report...";

task5->args0 = "Cleaning up temp files...";

task1->arg_count = 1;

task2->arg_count = 1;

task3->arg_count = 1;

task4->arg_count = 1;

task5->arg_count = 1;

// 设置依赖: extract → transform → load → send_report → cleanup

dag_set_dependency(task1, task2);

dag_set_dependency(task2, task3);

dag_set_dependency(task3, task4);

dag_set_dependency(task4, task5);

// 启动调度器

airflow_start(af);

// 等待任务执行

printf("等待任务执行...\n");

sleep(3);

// 打印状态

airflow_print_dag_status(af, "etl_pipeline");

airflow_stop(af);

free(af);

}

int main() {

test_airflow();

return 0;

}

```

---

三、编译和运行

```bash

gcc -o airflow airflow.c -lpthread

./airflow

```

---

四、Airflow vs 本实现

特性 本实现 Airflow

DAG定义 ✅ ✅

任务依赖 ✅ ✅

调度器 ✅ ✅

执行器 ✅ 基础 ✅ 多种

重试机制 ✅ ✅

Web UI ❌ ✅

数据库持久化 ❌ ✅

XCom ❌ ✅

---

五、总结

通过这篇文章，你学会了：

· Airflow的核心架构（Scheduler + Executor）

· DAG定义与任务创建

· 任务依赖关系

· 调度器实现

· 任务状态管理

· 重试机制

Airflow是任务编排的经典实现。掌握它，你就理解了数据管道调度的核心设计。

下一篇预告：《从零实现一个分布式数据仓库：Apache Hive的核心设计》

---

评论区分享一下你用Airflow编排过什么数据管道～