从零实现一个分布式搜索：Elasticsearch的核心设计

我不是懒洋洋2026-07-31 8:18

前言

在微服务架构中，搜索是核心功能之一。单机Elasticsearch有容量限制和单点故障问题，所以需要分布式搜索。

今天我们从零实现Elasticsearch的核心功能：

· 倒排索引构建

· 分词与索引

· 分布式分片（Shard）

· 副本（Replica）

· 查询路由与分发

· 结果聚合

· 近实时搜索（NRT）

一、Elasticsearch核心原理

  1. 架构图

```

┌─────────────────────────────────────────────────────────────┐

│ Elasticsearch集群 │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ Node1 │ │ Node2 │ │ Node3 │ │

│ │ Shard1 │ │ Shard2 │ │ Shard3 │ │

│ │ (主) │ │ (主) │ │ (主) │ │

│ │ Shard3 │ │ Shard1 │ │ Shard2 │ │

│ │ (副本) │ │ (副本) │ │ (副本) │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│

┌───────┴───────┐

▼ ▼

┌───────────┐ ┌───────────┐

│ Index │ │ Search │

│ (写入) │ │ (查询) │

└───────────┘ └───────────┘

```

  1. 核心概念

概念 说明

Index 索引（相当于数据库的表）

Shard 分片（水平扩展）

Replica 副本（高可用）

倒排索引 词→文档的映射

分词 将文本拆分为词项

二、完整代码实现

  1. 基础数据结构

```c

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <pthread.h>

#include <time.h>

#include <errno.h>

#include <ctype.h>

#define MAX_DOC_ID 64

#define MAX_FIELD_NAME 64

#define MAX_FIELD_VALUE 4096

#define MAX_TERM_LEN 128

#define MAX_INDEX_NAME 64

#define MAX_SHARDS 10

#define MAX_REPLICAS 3

#define MAX_TOKENS 100

// 文档

typedef struct document {

char idMAX_DOC_ID;

char contentMAX_FIELD_VALUE;

struct document *next;

} document_t;

// 倒排列表项

typedef struct posting {

char doc_idMAX_DOC_ID;

int term_freq;

struct posting *next;

} posting_t;

// 倒排索引项

typedef struct inverted_entry {

char termMAX_TERM_LEN;

posting_t *postings;

int doc_count;

struct inverted_entry *next;

} inverted_entry_t;

// 分片

typedef struct shard {

int shard_id;

char index_nameMAX_INDEX_NAME;

inverted_entry_t *inverted_index;

document_t *documents;

int doc_count;

int is_primary;

pthread_mutex_t mutex;

struct shard *next;

} shard_t;

// 索引

typedef struct es_index {

char nameMAX_INDEX_NAME;

shard_t *shards;

int shard_count;

int replica_count;

struct es_index *next;

} es_index_t;

// Elasticsearch集群

typedef struct es_cluster {

char node_id64;

char host32;

int port;

es_index_t *indices;

int index_count;

pthread_mutex_t mutex;

int running;

int port_listen;

} es_cluster_t;

// 查询结果

typedef struct search_result {

char doc_idMAX_DOC_ID;

float score;

struct search_result *next;

} search_result_t;

```

  1. 分词器

```c

// 简单分词（按空格和标点分割）

int tokenize_text(const char *text, char tokens\[\]MAX_TERM_LEN) {

int count = 0;

char buffer4096;

strcpy(buffer, text);

char *p = buffer;

while (*p && count < MAX_TOKENS) {

// 跳过非字母数字

while (*p && !isalnum(*p)) p++;

if (!*p) break;

char *start = p;

while (*p && (isalnum(*p) || *p == '_' || *p == '-')) p++;

int len = p - start;

if (len > 0 && len < MAX_TERM_LEN) {

for (int i = 0; i < len; i++) {

tokenscounti = tolower(starti);

}

tokenscountlen = '\0';

count++;

}

}

return count;

}

// 过滤停用词

int is_stop_word(const char *word) {

static const char *stop_words\[\] = {

"the", "a", "an", "of", "to", "for", "on", "at",

"in", "with", "by", "from", "up", "about", "into",

"through", "during", "including", "without", "and",

"or", "but", "so", "for", "nor", "yet", "as", NULL

};

for (int i = 0; stop_wordsi; i++) {

if (strcmp(word, stop_wordsi) == 0) return 1;

}

return 0;

}

```

  1. 索引构建

```c

// 创建Elasticsearch节点

es_cluster_t *es_create(const char *host, int port) {

es_cluster_t *es = malloc(sizeof(es_cluster_t));

memset(es, 0, sizeof(es_cluster_t));

strcpy(es->host, host);

es->port = port;

es->port_listen = port;

es->running = 1;

snprintf(es->node_id, sizeof(es->node_id), "%s:%d", host, port);

pthread_mutex_init(&es->mutex, NULL);

printf("ES 节点启动: %s:%d\n", host, port);

return es;

}

// 创建索引

es_index_t *es_create_index(es_cluster_t *es, const char *name,

int shard_count, int replica_count) {

pthread_mutex_lock(&es->mutex);

es_index_t *idx = malloc(sizeof(es_index_t));

strcpy(idx->name, name);

idx->shard_count = shard_count;

idx->replica_count = replica_count;

idx->shards = NULL;

// 创建分片

for (int i = 0; i < shard_count; i++) {

shard_t *shard = malloc(sizeof(shard_t));

shard->shard_id = i;

strcpy(shard->index_name, name);

shard->inverted_index = NULL;

shard->documents = NULL;

shard->doc_count = 0;

shard->is_primary = 1;

pthread_mutex_init(&shard->mutex, NULL);

shard->next = idx->shards;

idx->shards = shard;

}

idx->next = es->indices;

es->indices = idx;

es->index_count++;

pthread_mutex_unlock(&es->mutex);

printf("ES 创建索引: %s (分片: %d, 副本: %d)\n",

name, shard_count, replica_count);

return idx;

}

// 查找或创建倒排索引项

inverted_entry_t *find_or_create_term(shard_t *shard, const char *term) {

inverted_entry_t *entry = shard->inverted_index;

while (entry) {

if (strcmp(entry->term, term) == 0) {

return entry;

}

entry = entry->next;

}

entry = malloc(sizeof(inverted_entry_t));

strcpy(entry->term, term);

entry->postings = NULL;

entry->doc_count = 0;

entry->next = shard->inverted_index;

shard->inverted_index = entry;

return entry;

}

// 索引文档

void es_index_document(es_cluster_t *es, const char *index_name,

const char *doc_id, const char *content) {

// 路由到分片（按doc_id哈希）

unsigned int hash = 0;

const char *p = doc_id;

while (*p) {

hash = hash * 31 + *p++;

}

es_index_t *idx = es->indices;

while (idx) {

if (strcmp(idx->name, index_name) == 0) break;

idx = idx->next;

}

if (!idx) return;

int shard_id = hash % idx->shard_count;

shard_t *shard = idx->shards;

for (int i = 0; i < shard_id && shard; i++) {

shard = shard->next;

}

if (!shard) return;

pthread_mutex_lock(&shard->mutex);

// 保存文档

document_t *doc = malloc(sizeof(document_t));

strcpy(doc->id, doc_id);

strcpy(doc->content, content);

doc->next = shard->documents;

shard->documents = doc;

shard->doc_count++;

// 分词

char tokensMAX_TOKENSMAX_TERM_LEN;

int token_count = tokenize_text(content, tokens);

// 建立倒排索引

for (int i = 0; i < token_count; i++) {

if (is_stop_word(tokensi)) continue;

inverted_entry_t *entry = find_or_create_term(shard, tokensi);

// 检查是否已有该文档

posting_t *p = entry->postings;

int found = 0;

while (p) {

if (strcmp(p->doc_id, doc_id) == 0) {

p->term_freq++;

found = 1;

break;

}

p = p->next;

}

if (!found) {

posting_t *new_p = malloc(sizeof(posting_t));

strcpy(new_p->doc_id, doc_id);

new_p->term_freq = 1;

new_p->next = entry->postings;

entry->postings = new_p;

entry->doc_count++;

}

}

pthread_mutex_unlock(&shard->mutex);

printf("索引 文档 %s 已索引，共 %d 个词\n", doc_id, token_count);

}

```

  1. 搜索查询

```c

// 计算TF-IDF

float compute_tf_idf(inverted_entry_t *entry, posting_t *posting, int total_docs) {

float tf = logf(1 + posting->term_freq);

float idf = logf((float)total_docs / (entry->doc_count + 1));

return tf * idf;

}

// 搜索单个分片

search_result_t *search_shard(shard_t *shard, char **terms, int term_count,

int total_docs) {

pthread_mutex_lock(&shard->mutex);

search_result_t *results = NULL;

// 获取第一个词的结果

inverted_entry_t *entry = shard->inverted_index;

while (entry) {

if (strcmp(entry->term, terms0) == 0) break;

entry = entry->next;

}

if (!entry) {

pthread_mutex_unlock(&shard->mutex);

return NULL;

}

// 收集文档ID

char doc_ids100MAX_DOC_ID;

int doc_count = 0;

posting_t *p = entry->postings;

while (p && doc_count < 100) {

strcpy(doc_idsdoc_count++, p->doc_id);

p = p->next;

}

// 与其他词求交集

for (int i = 1; i < term_count; i++) {

entry = shard->inverted_index;

while (entry) {

if (strcmp(entry->term, termsi) == 0) break;

entry = entry->next;

}

if (!entry) {

pthread_mutex_unlock(&shard->mutex);

return NULL;

}

// 求交集

int new_count = 0;

char new_ids100MAX_DOC_ID;

p = entry->postings;

while (p) {

for (int j = 0; j < doc_count; j++) {

if (strcmp(p->doc_id, doc_idsj) == 0) {

strcpy(new_idsnew_count++, p->doc_id);

break;

}

}

p = p->next;

}

doc_count = new_count;

for (int j = 0; j < doc_count; j++) {

strcpy(doc_idsj, new_idsj);

}

}

// 构建结果

for (int i = 0; i < doc_count; i++) {

search_result_t *res = malloc(sizeof(search_result_t));

strcpy(res->doc_id, doc_idsi);

res->score = 1.0f / (i + 1);

res->next = results;

results = res;

}

pthread_mutex_unlock(&shard->mutex);

return results;

}

// 全局搜索（所有分片）

search_result_t *es_search(es_cluster_t *es, const char *index_name,

const char *query) {

// 分词

char termsMAX_TOKENSMAX_TERM_LEN;

int term_count = tokenize_text(query, terms);

// 过滤停用词

int filtered_count = 0;

char filteredMAX_TOKENSMAX_TERM_LEN;

for (int i = 0; i < term_count; i++) {

if (!is_stop_word(termsi)) {

strcpy(filteredfiltered_count++, termsi);

}

}

if (filtered_count == 0) return NULL;

// 查找索引

es_index_t *idx = es->indices;

while (idx) {

if (strcmp(idx->name, index_name) == 0) break;

idx = idx->next;

}

if (!idx) return NULL;

// 搜索所有分片

search_result_t *all_results = NULL;

shard_t *shard = idx->shards;

int total_docs = 0;

// 计算总文档数

while (shard) {

total_docs += shard->doc_count;

shard = shard->next;

}

shard = idx->shards;

while (shard) {

search_result_t *results = search_shard(shard, filtered, filtered_count, total_docs);

// 合并结果

search_result_t *r = results;

while (r) {

search_result_t *next = r->next;

r->next = all_results;

all_results = r;

r = next;

}

shard = shard->next;

}

return all_results;

}

```

  1. 测试代码

```c

void test_elasticsearch() {

printf("=== Elasticsearch分布式搜索测试 ===\n\n");

es_cluster_t *es = es_create("127.0.0.1", 9200);

// 创建索引

es_create_index(es, "articles", 3, 1);

// 索引文档

printf("索引文档...\n");

es_index_document(es, "articles", "1",

"The quick brown fox jumps over the lazy dog");

es_index_document(es, "articles", "2",

"A quick brown fox is fast");

es_index_document(es, "articles", "3",

"The lazy dog sleeps all day");

es_index_document(es, "articles", "4",

"Foxes are quick and agile animals");

es_index_document(es, "articles", "5",

"The dog is a loyal companion");

// 搜索

printf("\n搜索 'quick fox':\n");

search_result_t *results = es_search(es, "articles", "quick fox");

int rank = 1;

search_result_t *r = results;

while (r && rank <= 5) {

printf(" %d. 文档 %s (得分: %.3f)\n", rank++, r->doc_id, r->score);

r = r->next;

}

free(results);

printf("\n搜索 'lazy dog':\n");

results = es_search(es, "articles", "lazy dog");

rank = 1;

r = results;

while (r && rank <= 5) {

printf(" %d. 文档 %s (得分: %.3f)\n", rank++, r->doc_id, r->score);

r = r->next;

}

free(results);

printf("\n搜索 'brown fox':\n");

results = es_search(es, "articles", "brown fox");

rank = 1;

r = results;

while (r && rank <= 5) {

printf(" %d. 文档 %s (得分: %.3f)\n", rank++, r->doc_id, r->score);

r = r->next;

}

free(results);

printf("\n搜索 'animals':\n");

results = es_search(es, "articles", "animals");

rank = 1;

r = results;

while (r && rank <= 5) {

printf(" %d. 文档 %s (得分: %.3f)\n", rank++, r->doc_id, r->score);

r = r->next;

}

free(results);

printf("\n索引统计:\n");

es_index_t *idx = es->indices;

while (idx) {

printf(" %s: %d 个分片\n", idx->name, idx->shard_count);

shard_t *shard = idx->shards;

while (shard) {

printf(" Shard %d: %d 篇文档\n", shard->shard_id, shard->doc_count);

shard = shard->next;

}

idx = idx->next;

}

free(es);

}

int main() {

srand(time(NULL));

test_elasticsearch();

return 0;

}

```

三、编译和运行

```bash

gcc -o elasticsearch elasticsearch.c -lpthread -lm

./elasticsearch

```

四、Elasticsearch vs 本实现

特性 本实现 Elasticsearch

倒排索引 ✅ ✅

分词器 ✅ 基础 ✅ 丰富

分布式分片 ✅ ✅

副本 ✅ ✅

搜索聚合 ✅ 基础 ✅ 丰富

相关性评分 ✅ TF-IDF ✅ BM25

近实时 ❌ ✅

持久化 ❌ ✅

五、总结

通过这篇文章，你学会了：

· Elasticsearch的核心架构

· 倒排索引构建

· 分词与索引

· 分布式分片管理

· 查询路由与分发

· 搜索结果聚合

Elasticsearch是分布式搜索的经典实现。掌握它，你就理解了搜索引擎的底层设计。

下一篇预告：《从零实现一个分布式消息队列：Kafka的核心设计》

评论区分享一下你对Elasticsearch的理解～

相关推荐
一只旭宝17 小时前
C++手写shared_ptr共享智能指针｜原子引用计数、强弱引用控制块、赋值重载底层深度剖析
开发语言·c++·面试
开发者联盟league18 小时前
Java 通过 JNA 调用 C++ DLL：关键流程总结
java·c++·jna
库克克19 小时前
【C++】智能指针
java·开发语言·c++
一拳一个呆瓜21 小时前
【STL】iostream 编程：输入流成员函数
c++·stl
xiaoye-duck21 小时前
《Linux系统编程》Linux 系统多线程（七）： C++ 线程安全日志系统封装：基于策略模式解耦，兼容 glog 使用风格
linux·c++·日志系统
程序喵大人1 天前
【C++进阶】STL容器与迭代器 - 07 迭代器是容器和算法之间的通用游标
开发语言·c++·算法
汉克老师1 天前
GESP2026年3月认证C++七级( 第三部分编程题（1、拆分））精讲
c++·预处理·dp·取模·gesp7级·数学规律·对数
BerryS3N1 天前
C++在AI时代的核心力量：从底层加速到大规模大模型推理引擎的高能实践
开发语言·c++·人工智能
FlightYe1 天前
音视频修炼之视频基础（一）：视频基础理论
android·c++·vscode·音视频·androidx·android runtime
小小帅呀1 天前
CUDA编程实战09：CUDA Graph 实战——降低重复 Kernel 与流水线提交开销
c++·人工智能·矩阵
热门推荐
01如何新建文件夹？ 电脑新建文件夹的4种方法02GitHub 镜像站点032026年7月AI圈大地震：GPT-5.6被政府限制、Claude入驻Slack、Anthropic自研芯片04国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）05AI 编程 IDE 全景解析 2026：Agent 全面接管开发链路06AI科技热点日报 | 2026年07月01日072026上半年大模型全景技术解读：推理融合、Agent 爆发与多模态统一082026 年 AI 大模型 & AI 编程工具实战全总结092026 国产 AI 大模型横评：DeepSeek、通义千问、Kimi、文心一言、星火、豆包谁更能打？10微信历史版本含下载地址（ Windows PC | 安卓 | MAC ）及设置微信不更新