# es_demo.py
import json
from dataclasses import dataclass
from typing import Optional, List, Dict, Any
from elasticsearch import Elasticsearch
import uuid
from day05_Info_model.meta_demo.meta_config import TableConfig, MetaConfig, MetricConfig, ColumnConfig
# ========== 数据模型（与MySQL相同，略） ==========
# ...（同上的 ColumnConfig, TableConfig, MetricConfig, MetaConfig）
# ========== Elasticsearch 仓储实现 ==========
class ElasticsearchMetaRepository:
def __init__(self, hosts: List[str] = ["http://localhost:9200"], index_name: str = "meta_configs_demo"):
self.client = Elasticsearch(hosts=hosts)
self.index_name = index_name
self._init_index()
def _init_index(self):
"""初始化索引"""
if not self.client.indices.exists(index=self.index_name):
mapping = {
"mappings": {
"properties": {
"config_data": {
"type": "text",
"fields": {
"keyword": {"type": "keyword"}
}
},
"table_names": {
"type": "text",
"fields": {
"keyword": {"type": "keyword"}
}
},
"metric_names": {
"type": "text",
"fields": {
"keyword": {"type": "keyword"}
}
},
"has_tables": {"type": "boolean"},
"has_metrics": {"type": "boolean"},
"created_at": {"type": "date"},
"updated_at": {"type": "date"}
}
}
}
self.client.indices.create(index=self.index_name, **mapping)
print("✅ ES索引创建成功")
else:
print("✅ ES索引已存在")
def _prepare_document(self, entity: MetaConfig, doc_id: Optional[str] = None) -> Dict[str, Any]:
"""准备ES文档"""
doc = {
"config_data": json.dumps(entity.to_dict(), ensure_ascii=False),
"table_names": [t.name for t in entity.tables] if entity.tables else [],
"metric_names": [m.name for m in entity.metrics] if entity.metrics else [],
"has_tables": bool(entity.tables),
"has_metrics": bool(entity.metrics)
}
if doc_id:
doc["_id"] = doc_id
return doc
def _doc_to_entity(self, doc: Dict[str, Any]) -> MetaConfig:
"""ES文档转实体"""
data = json.loads(doc["_source"]["config_data"])
return MetaConfig.from_dict(data)
# ========== CRUD 操作 ==========
def create(self, entity: MetaConfig) -> str:
"""创建配置，返回ID"""
doc_id = str(uuid.uuid4())
doc = self._prepare_document(entity)
response = self.client.index(
index=self.index_name,
id=doc_id,
document=doc
)
print(f"✅ ES创建成功，ID: {doc_id}")
return doc_id
def get_by_id(self, id: str) -> Optional[MetaConfig]:
"""根据ID获取配置"""
try:
result = self.client.get(index=self.index_name, id=id)
if result and result["found"]:
return self._doc_to_entity(result)
except Exception as e:
print(f"❌ 查询失败: {e}")
return None
def get_all(self, size: int = 100) -> List[MetaConfig]:
"""获取所有配置"""
response = self.client.search(
index=self.index_name,
query={"match_all": {}},
size=size
)
return [self._doc_to_entity(hit) for hit in response["hits"]["hits"]]
def update(self, id: str, entity: MetaConfig) -> bool:
"""更新配置"""
try:
# 检查是否存在
existing = self.client.get(index=self.index_name, id=id)
if not existing or not existing["found"]:
print(f"❌ ID {id} 不存在")
return False
doc = self._prepare_document(entity)
self.client.index(
index=self.index_name,
id=id,
document=doc
)
print(f"✅ ES更新成功，ID: {id}")
return True
except Exception as e:
print(f"❌ 更新失败: {e}")
return False
def delete(self, id: str) -> bool:
"""删除配置"""
try:
response = self.client.delete(index=self.index_name, id=id)
success = response["result"] == "deleted"
if success:
print(f"✅ ES删除成功，ID: {id}")
else:
print(f"❌ ID {id} 不存在")
return success
except Exception as e:
print(f"❌ 删除失败: {e}")
return False
def search(self, query: str, size: int = 100, **kwargs) -> List[MetaConfig]:
"""全文搜索"""
search_fields = kwargs.get("fields", ["table_names", "metric_names", "config_data"])
must = [
{
"multi_match": {
"query": query,
"fields": search_fields,
"operator": "or"
}
}
]
# 添加过滤器
if "has_tables" in kwargs:
must.append({"term": {"has_tables": kwargs["has_tables"]}})
if "has_metrics" in kwargs:
must.append({"term": {"has_metrics": kwargs["has_metrics"]}})
response = self.client.search(
index=self.index_name,
query={"bool": {"must": must}},
size=size
)
return [self._doc_to_entity(hit) for hit in response["hits"]["hits"]]
def search_tables(self, query: str) -> List[MetaConfig]:
"""搜索包含特定表的配置"""
return self.search(query, fields=["table_names"])
def search_metrics(self, query: str) -> List[MetaConfig]:
"""搜索包含特定指标的配置"""
return self.search(query, fields=["metric_names"])
def search_exact(self, field: str, value: str) -> List[MetaConfig]:
"""精确搜索"""
response = self.client.search(
index=self.index_name,
query={"term": {field: value}},
size=100
)
return [self._doc_to_entity(hit) for hit in response["hits"]["hits"]]
# ========== 演示函数 ==========
def es_demo():
print("=" * 60)
print("🔷 Elasticsearch 元数据管理演示")
print("=" * 60)
# 1. 初始化
repo = ElasticsearchMetaRepository(
hosts=["http://localhost:9200"],
index_name="meta_configs"
)
# 2. 准备测试数据
print("\n📦 准备测试数据...")
columns = [
ColumnConfig(
name="order_id",
role="primary_key",
description="订单唯一标识",
alias=["订单ID", "单号"],
sync=False
),
ColumnConfig(
name="order_date",
role="dimension",
description="订单创建日期",
alias=["日期", "下单日期"],
sync=True
),
ColumnConfig(
name="order_status",
role="dimension",
description="订单状态，如已支付、已发货等",
alias=["状态", "订单状态"],
sync=True
)
]
tables = [
TableConfig(
name="fact_order",
role="fact",
description="订单事实表，存储所有订单交易数据",
columns=columns
)
]
metrics = [
MetricConfig(
name="GMV",
description="全称Gross Merchandise value,表示所有订单的成交金额总和",
relevant_columns=["fact_order.order_amount"],
alias=["成交总额", "订单总额"]
),
MetricConfig(
name="AOV",
description="全称Average Order value,表示所有订单的成交金额总的平均值",
relevant_columns=["fact_order.order_amount", "fact_order.order_id"],
alias=["平均单价", "平均订单金额"]
)
]
meta_config = MetaConfig(tables=tables, metrics=metrics)
# 3. 创建
print("\n📝 创建配置...")
config_id = repo.create(meta_config)
# 4. 查询
print("\n🔍 根据ID查询...")
retrieved = repo.get_by_id(config_id)
if retrieved:
print(f" 查询成功!")
print(f" 表数量: {len(retrieved.tables) if retrieved.tables else 0}")
print(f" 指标数量: {len(retrieved.metrics) if retrieved.metrics else 0}")
# 5. 全文搜索
print("\n🔍 全文搜索 '订单 交易'...")
search_results = repo.search("订单 交易")
print(f" 找到 {len(search_results)} 个配置")
# 显示结果详情
for i, config in enumerate(search_results[:3], 1):
table_names = [t.name for t in config.tables] if config.tables else []
metric_names = [m.name for m in config.metrics] if config.metrics else []
print(f" {i}. 表: {table_names}, 指标: {metric_names}")
# 6. 按表名搜索
print("\n🔍 按表名搜索 'fact_order'...")
table_results = repo.search_tables("fact_order")
print(f" 找到 {len(table_results)} 个配置")
# 7. 按指标名搜索
print("\n🔍 按指标名搜索 'GMV'...")
metric_results = repo.search_metrics("GMV")
print(f" 找到 {len(metric_results)} 个配置")
# 8. 创建第二个配置（用于演示多条件搜索）
print("\n📝 创建第二个配置...")
columns2 = [
ColumnConfig(
name="product_id",
role="primary_key",
description="商品唯一标识",
alias=["商品ID", "产品ID"],
sync=False
),
ColumnConfig(
name="product_name",
role="dimension",
description="商品名称",
alias=["商品名", "产品名"],
sync=True
)
]
tables2 = [
TableConfig(
name="dim_product",
role="dim",
description="商品维度表，存储商品信息",
columns=columns2
)
]
meta_config2 = MetaConfig(tables=tables2, metrics=None)
config_id2 = repo.create(meta_config2)
# 9. 过滤搜索（只搜索有指标的配置）
print("\n🔍 过滤搜索：只搜索有指标的配置...")
filtered_results = repo.search("order", has_metrics=True)
print(f" 找到 {len(filtered_results)} 个配置")
# 10. 精确搜索
print("\n🔍 精确搜索：表名 = 'dim_product'...")
exact_results = repo.search_exact("table_names", "dim_product")
print(f" 找到 {len(exact_results)} 个配置")
# 11. 获取所有
print("\n📋 获取所有配置...")
all_configs = repo.get_all()
print(f" 共有 {len(all_configs)} 个配置")
# 12. 删除
print("\n🗑️ 删除配置...")
repo.delete(config_id)
repo.delete(config_id2)
print(" 删除成功!")
print("\n" + "=" * 60)
print("✅ Elasticsearch演示完成!")
print("=" * 60)
if __name__ == "__main__":
es_demo()
'''
============================================================
🔷 Elasticsearch 元数据管理演示
============================================================
✅ ES索引创建成功
📦 准备测试数据...
📝 创建配置...
✅ ES创建成功，ID: 6a7522e1-c660-40b5-895e-0453c9a5fe7b
🔍 根据ID查询...
查询成功!
表数量: 1
指标数量: 2
🔍 全文搜索 '订单 交易'...
找到 0 个配置
🔍 按表名搜索 'fact_order'...
找到 0 个配置
🔍 按指标名搜索 'GMV'...
找到 0 个配置
📝 创建第二个配置...
✅ ES创建成功，ID: ae6624b5-9f30-4a8d-97b8-852eff053683
🔍 过滤搜索：只搜索有指标的配置...
找到 0 个配置
🔍 精确搜索：表名 = 'dim_product'...
找到 0 个配置
📋 获取所有配置...
共有 0 个配置
🗑️ 删除配置...
✅ ES删除成功，ID: 6a7522e1-c660-40b5-895e-0453c9a5fe7b
✅ ES删除成功，ID: ae6624b5-9f30-4a8d-97b8-852eff053683
删除成功!
============================================================
✅ Elasticsearch演示完成!
============================================================
'''