物联网协议网关设计：多协议融合与数据转换的工程实战
引言
在真实的物联网部署场景中，一个系统往往需要同时接入多种不同通信协议的设备------MQTT的传感器、Modbus的PLC、CoAP的低功耗节点、HTTP的第三方API。如何让这些"语言不通"的设备协同工作，是协议网关要解决的核心问题。
沧州虎王科技在智慧工厂项目中，需要同时接入Modbus RTU的工业传感器、MQTT的环境监测节点和HTTP的第三方数据源。通过自主设计的ESP32协议网关，我们实现了多协议统一接入和实时数据转换，将系统接入效率提升了3倍。本文将完整分享这一协议网关的设计思路与实现方案。
一、协议网关核心架构
1.1 设计目标
协议网关需要实现以下核心能力：
- 协议适配：支持MQTT、Modbus、CoAP、HTTP等多种协议接入
- 数据转换：将不同协议的数据格式统一转换为标准JSON
- 路由转发：根据规则将数据路由到目标系统
- 连接管理：管理所有设备的连接状态和心跳
- 缓冲重试：网络异常时缓存数据，恢复后重发
1.2 整体架构
┌─────────────────────────────────────────────────┐
│ 协议网关架构 │
├─────────────────────────────────────────────────┤
│ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ Modbus │ │ MQTT │ │ HTTP │ │
│ │ Adapter │ │ Adapter │ │ Adapter │ │
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
│ │ │ │ │
│ ▼ ▼ ▼ │
│ ┌─────────────────────────────────────┐ │
│ │ 消息总线（内部MQTT） │ │
│ └─────────────────┬───────────────────┘ │
│ │ │
│ ┌────────────┼────────────┐ │
│ ▼ ▼ ▼ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ 数据 │ │ 路由 │ │ 规则 │ │
│ │ 转换器 │ │ 引擎 │ │ 引擎 │ │
│ └──────────┘ └──────────┘ └──────────┘ │
│ │
└─────────────────────────────────────────────────┘1.3 模块职责
每个适配器负责一种协议的接入，将协议特有的数据格式转换为统一的内部消息格式后发布到消息总线。消息总线上的消息经过数据转换器、路由引擎和规则引擎处理后，转发到目标系统。
二、统一数据模型设计
2.1 内部消息格式
所有协议适配器都将数据转换为统一的内部消息格式：
json
{
"msg_id": "550e8400-e29b-41d4-a716-446655440000",
"timestamp": 1723277400000,
"source": {
"protocol": "modbus",
"device_id": "sensor_001",
"address": "192.168.1.100:502",
"unit_id": 1
},
"payload": {
"type": "telemetry",
"data": {
"temperature": 25.6,
"humidity": 60.2,
"pressure": 1013.25
}
},
"metadata": {
"quality": "good",
"raw_values": [25.6, 60.2, 1013.25]
}
}2.2 协议适配器接口
python
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Dict, Any, Optional
import uuid
import time
@dataclass
class InternalMessage:
msg_id: str
timestamp: int
source_protocol: str
source_device_id: str
source_address: str
msg_type: str # telemetry, command, event, status
data: Dict[str, Any]
metadata: Dict[str, Any] = None
@staticmethod
def create(protocol: str, device_id: str, msg_type: str,
data: dict, address: str = ""):
return InternalMessage(
msg_id=str(uuid.uuid4()),
timestamp=int(time.time() * 1000),
source_protocol=protocol,
source_device_id=device_id,
source_address=address,
msg_type=msg_type,
data=data,
metadata={}
)
def to_json(self) -> dict:
return {
"msg_id": self.msg_id,
"timestamp": self.timestamp,
"source": {
"protocol": self.source_protocol,
"device_id": self.source_device_id,
"address": self.source_address
},
"payload": {
"type": self.msg_type,
"data": self.data
},
"metadata": self.metadata or {}
}
class ProtocolAdapter(ABC):
"""协议适配器抽象基类"""
def __init__(self, name: str, config: dict):
self.name = name
self.config = config
self.devices: Dict[str, dict] = {}
self.message_callback = None
@abstractmethod
async def start(self):
"""启动适配器"""
pass
@abstractmethod
async def stop(self):
"""停止适配器"""
pass
@abstractmethod
async def send_command(self, device_id: str, command: dict):
"""向设备发送命令"""
pass
def on_message(self, callback):
"""注册消息回调"""
self.message_callback = callback
def _emit_message(self, message: InternalMessage):
"""向消息总线发布消息"""
if self.message_callback:
self.message_callback(message)三、Modbus适配器实现
3.1 Modbus TCP适配器
Modbus是工业领域最常用的通信协议之一。以下是基于pymodbus库的Modbus TCP适配器实现：
python
from pymodbus.client import ModbusTcpClient
from pymodbus.exceptions import ModbusException
import asyncio
import struct
class ModbusTcpAdapter(ProtocolAdapter):
def __init__(self, config: dict):
super().__init__("modbus_tcp", config)
self.clients: Dict[str, ModbusTcpClient] = {}
self.poll_interval = config.get("poll_interval", 1.0)
self.register_map = config.get("register_map", {})
async def start(self):
"""启动Modbus适配器"""
# 初始化所有设备连接
for device_id, device_config in self.config["devices"].items():
client = ModbusTcpClient(
device_config["host"],
port=device_config.get("port", 502),
timeout=device_config.get("timeout", 3)
)
if client.connect():
self.clients[device_id] = client
self.devices[device_id] = device_config
logger.info(f"Modbus device connected: {device_id}")
else:
logger.error(f"Modbus device connection failed: {device_id}")
# 启动轮询任务
asyncio.create_task(self._poll_loop())
async def _poll_loop(self):
"""轮询设备寄存器"""
while True:
for device_id, client in list(self.clients.items()):
try:
await self._read_device(device_id, client)
except Exception as e:
logger.error(f"Modbus read error for {device_id}: {e}")
# 尝试重连
await self._reconnect(device_id)
await asyncio.sleep(self.poll_interval)
async def _read_device(self, device_id: str, client: ModbusTcpClient):
"""读取设备所有寄存器"""
device_config = self.devices[device_id]
unit_id = device_config.get("unit_id", 1)
register_groups = device_config.get("registers", [])
all_values = {}
for group in register_groups:
reg_type = group["type"] # holding, input, coil, discrete
start_addr = group["start"]
count = group["count"]
try:
if reg_type == "holding":
result = client.read_holding_registers(
start_addr, count, slave=unit_id)
elif reg_type == "input":
result = client.read_input_registers(
start_addr, count, slave=unit_id)
elif reg_type == "coil":
result = client.read_coils(
start_addr, count, slave=unit_id)
elif reg_type == "discrete":
result = client.read_discrete_inputs(
start_addr, count, slave=unit_id)
else:
continue
if result.isError():
logger.warning(f"Modbus error reading {device_id}: {result}")
continue
# 转换寄存器值为工程值
raw_values = result.registers if hasattr(result, 'registers') \
else result.bits
converted = self._convert_registers(
raw_values, group.get("conversion", {}))
all_values.update(converted)
except ModbusException as e:
logger.error(f"Modbus exception for {device_id}: {e}")
continue
if all_values:
msg = InternalMessage.create(
protocol="modbus_tcp",
device_id=device_id,
msg_type="telemetry",
data=all_values,
address=f"{self.devices[device_id]['host']}:{self.devices[device_id].get('port', 502)}"
)
msg.metadata["quality"] = "good"
msg.metadata["unit_id"] = unit_id
self._emit_message(msg)
def _convert_registers(self, raw_values: list,
conversion: dict) -> dict:
"""将原始寄存器值转换为工程值"""
result = {}
for name, conv_config in conversion.items():
offset = conv_config.get("offset", 0)
scale = conv_config.get("scale", 1.0)
data_type = conv_config.get("type", "uint16")
precision = conv_config.get("precision", 2)
if data_type == "uint16":
raw = raw_values[offset]
value = raw * scale
elif data_type == "int16":
raw = raw_values[offset]
if raw > 32767:
raw -= 65536
value = raw * scale
elif data_type == "uint32":
# 大端序：高位在前
raw = (raw_values[offset] << 16) | raw_values[offset + 1]
value = raw * scale
elif data_type == "float32":
# 32位浮点数，占2个寄存器
raw_bytes = struct.pack('>HH',
raw_values[offset],
raw_values[offset + 1])
value = struct.unpack('>f', raw_bytes)[0]
value = round(value, precision)
elif data_type == "bool":
value = bool(raw_values[offset])
else:
value = raw_values[offset]
result[name] = value
return result
async def send_command(self, device_id: str, command: dict):
"""向Modbus设备写入寄存器"""
if device_id not in self.clients:
return {"error": "device not connected"}
client = self.clients[device_id]
unit_id = self.devices[device_id].get("unit_id", 1)
addr = command.get("address")
value = command.get("value")
reg_type = command.get("type", "holding")
try:
if reg_type == "holding":
client.write_register(addr, int(value), slave=unit_id)
elif reg_type == "coil":
client.write_coil(addr, bool(value), slave=unit_id)
elif reg_type == "holding_multi":
client.write_registers(addr, [int(v) for v in value],
slave=unit_id)
return {"status": "success"}
except ModbusException as e:
return {"status": "error", "message": str(e)}
async def _reconnect(self, device_id: str):
"""重连设备"""
config = self.devices[device_id]
client = ModbusTcpClient(
config["host"],
port=config.get("port", 502)
)
if client.connect():
self.clients[device_id] = client
logger.info(f"Modbus device reconnected: {device_id}")
else:
logger.error(f"Modbus reconnect failed: {device_id}")
async def stop(self):
"""停止适配器"""
for device_id, client in self.clients.items():
client.close()
self.clients.clear()四、MQTT适配器实现
4.1 MQTT适配器
python
import asyncio
import json
import paho.mqtt.client as mqtt
class MqttAdapter(ProtocolAdapter):
def __init__(self, config: dict):
super().__init__("mqtt", config)
self.client = None
self.subscriptions = config.get("subscriptions", [])
async def start(self):
"""启动MQTT适配器"""
self.client = mqtt.Client(
client_id=self.config.get("client_id", "gateway_mqtt"),
clean_session=True
)
# 设置TLS（如果配置了）
if self.config.get("tls", False):
self.client.tls_set(
ca_certs=self.config.get("ca_cert"),
certfile=self.config.get("client_cert"),
keyfile=self.config.get("client_key")
)
# 设置用户名密码
if self.config.get("username"):
self.client.username_pw_set(
self.config["username"],
self.config.get("password", "")
)
# 注册回调
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
self.client.on_disconnect = self._on_disconnect
# 连接
self.client.connect(
self.config["host"],
self.config.get("port", 1883),
self.config.get("keepalive", 60)
)
# 在单独线程中运行MQTT循环
self.client.loop_start()
def _on_connect(self, client, userdata, flags, rc):
"""MQTT连接成功回调"""
if rc == 0:
logger.info("MQTT adapter connected")
for sub in self.subscriptions:
topic = sub["topic"]
qos = sub.get("qos", 0)
client.subscribe(topic, qos)
logger.info(f"Subscribed to: {topic}")
else:
logger.error(f"MQTT connection failed, rc={rc}")
def _on_message(self, client, userdata, msg):
"""收到MQTT消息"""
try:
payload = json.loads(msg.payload.decode())
# 从topic提取device_id
# topic格式: devices/{device_id}/telemetry
parts = msg.topic.split('/')
device_id = parts[1] if len(parts) > 1 else "unknown"
message = InternalMessage.create(
protocol="mqtt",
device_id=device_id,
msg_type="telemetry",
data=payload,
address=msg.topic
)
message.metadata["qos"] = msg.qos
message.metadata["retain"] = msg.retain
self._emit_message(message)
except json.JSONDecodeError:
logger.error(f"Invalid JSON in MQTT message: {msg.topic}")
except Exception as e:
logger.error(f"MQTT message processing error: {e}")
def _on_disconnect(self, client, userdata, rc):
"""MQTT断开连接回调"""
if rc != 0:
logger.warning(f"MQTT unexpected disconnect, rc={rc}")
async def send_command(self, device_id: str, command: dict):
"""通过MQTT发送命令"""
topic = f"devices/{device_id}/command"
payload = json.dumps(command)
result = self.client.publish(topic, payload, qos=1)
return {"status": "published", "mid": result.mid}
async def stop(self):
"""停止适配器"""
if self.client:
self.client.loop_stop()
self.client.disconnect()五、数据转换引擎
5.1 转换规则配置
数据转换引擎负责将不同协议的数据格式统一转换。使用声明式的转换规则：
python
class DataTransformer:
def __init__(self, config: dict):
self.rules = config.get("transform_rules", {})
self.unit_conversions = {
"c_to_f": lambda c: c * 9/5 + 32,
"f_to_c": lambda f: (f - 32) * 5/9,
"psi_to_kpa": lambda psi: psi * 6.89476,
"kpa_to_hpa": lambda kpa: kpa * 10,
"rpm_to_rps": lambda rpm: rpm / 60,
}
def transform(self, message: InternalMessage) -> InternalMessage:
"""根据规则转换消息"""
device_rules = self.rules.get(message.source_device_id, {})
if not device_rules:
return message # 无转换规则，原样返回
transformed_data = {}
for key, value in message.data.items():
rule = device_rules.get(key, {})
# 单位转换
if "unit_convert" in rule:
converter = self.unit_conversions.get(rule["unit_convert"])
if converter:
value = converter(value)
# 线性变换: y = ax + b
if "scale" in rule or "offset" in rule:
scale = rule.get("scale", 1.0)
offset = rule.get("offset", 0.0)
value = value * scale + offset
# 取整
if "round" in rule:
value = round(value, rule["round"])
# 重命名
new_key = rule.get("rename", key)
transformed_data[new_key] = value
message.data = transformed_data
message.metadata["transformed"] = True
return message5.2 转换规则示例
json
{
"transform_rules": {
"sensor_001": {
"temperature": {
"unit_convert": "c_to_f",
"rename": "temp_fahrenheit",
"round": 1
},
"pressure": {
"unit_convert": "kpa_to_hpa",
"rename": "pressure_hpa",
"round": 2
},
"raw_count": {
"scale": 0.1,
"offset": -50,
"rename": "distance_mm",
"round": 0
}
}
}
}六、ESP32轻量级网关实现
6.1 ESP32作为协议网关
在资源受限的场景下，ESP32本身也可以作为轻量级协议网关使用。以下是ESP32同时接入Modbus RTU设备和MQTT的简化实现：
c
#include "modbus_rtu.h"
#include "mqtt_client.h"
#include "cJSON.h"
#define MAX_MODBUS_DEVICES 8
#define MODBUS_UART_PORT UART_NUM_2
#define MODBUS_RS485_PIN 4
typedef struct {
uint8_t unit_id;
uint16_t start_addr;
uint16_t reg_count;
char device_name[32];
bool online;
} modbus_device_t;
static modbus_device_t modbus_devices[MAX_MODBUS_DEVICES];
static int modbus_device_count = 0;
static esp_mqtt_client_handle_t mqtt_client;
// 初始化Modbus RTU
void modbus_rtu_init(void) {
uart_config_t uart_config = {
.baud_rate = 9600,
.data_bits = UART_DATA_8_BITS,
.parity = UART_PARITY_DISABLE,
.stop_bits = UART_STOP_BITS_1,
.flow_ctrl = UART_HW_FLOWCTRL_DISABLE,
};
uart_param_config(MODBUS_UART_PORT, &uart_config);
uart_set_pin(MODBUS_UART_PORT, 17, 16,
MODBUS_RS485_PIN, UART_PIN_NO_CHANGE);
uart_set_mode(MODBUS_UART_PORT, UART_MODE_RS485_HALF_DUPLEX);
uart_driver_install(MODBUS_UART_PORT, 256, 256, 0, NULL, 0);
}
// 读取Modbus保持寄存器
esp_err_t modbus_read_holding(uint8_t unit_id,
uint16_t start_addr,
uint16_t count,
uint16_t *registers) {
uint8_t request[8];
request[0] = unit_id;
request[1] = 0x03; // 读保持寄存器功能码
request[2] = (start_addr >> 8) & 0xFF;
request[3] = start_addr & 0xFF;
request[4] = (count >> 8) & 0xFF;
request[5] = count & 0xFF;
// 计算CRC
uint16_t crc = modbus_crc(request, 6);
request[6] = crc & 0xFF;
request[7] = (crc >> 8) & 0xFF;
// 发送请求
uart_write_bytes(MODBUS_UART_PORT, (const char*)request, 8);
// 等待响应
uint8_t response[256];
int len = uart_read_bytes(MODBUS_UART_PORT, response,
3 + count * 2 + 2,
pdMS_TO_TICKS(1000));
if (len < 5) return ESP_ERR_TIMEOUT;
if (response[0] != unit_id) return ESP_ERR_INVALID_RESPONSE;
if (response[1] & 0x80) return ESP_ERR_INVALID_STATE; // 异常响应
// 提取寄存器值
for (int i = 0; i < count; i++) {
registers[i] = (response[3 + i*2] << 8) | response[4 + i*2];
}
return ESP_OK;
}
// Modbus到MQTT的数据转换与转发
void modbus_to_mqtt_task(void *pvParameters) {
uint16_t registers[32];
while (1) {
for (int i = 0; i < modbus_device_count; i++) {
modbus_device_t *dev = &modbus_devices[i];
esp_err_t ret = modbus_read_holding(
dev->unit_id, dev->start_addr,
dev->reg_count, registers);
if (ret == ESP_OK) {
dev->online = true;
// 构建JSON消息
cJSON *root = cJSON_CreateObject();
cJSON *data = cJSON_CreateObject();
cJSON_AddNumberToObject(data, "reg_0", registers[0]);
cJSON_AddNumberToObject(data, "reg_1", registers[1]);
cJSON_AddNumberToObject(data, "reg_2", registers[2]);
cJSON_AddStringToObject(root, "device", dev->device_name);
cJSON_AddItemToObject(root, "data", data);
cJSON_AddNumberToObject(root, "timestamp",
esp_timer_get_time() / 1000);
char *json_str = cJSON_PrintUnformatted(root);
// 发布到MQTT
char topic[128];
snprintf(topic, sizeof(topic),
"devices/%s/telemetry",
dev->device_name);
esp_mqtt_client_publish(mqtt_client, topic, json_str,
0, 1, 0);
free(json_str);
cJSON_Delete(root);
} else {
dev->online = false;
ESP_LOGW(TAG, "Modbus read failed: %s, unit=%d",
esp_err_to_name(ret), dev->unit_id);
}
vTaskDelay(pdMS_TO_TICKS(100)); // 设备间间隔
}
vTaskDelay(pdMS_TO_TICKS(5000)); // 5秒轮询周期
}
}
// 从MQTT接收命令并转发到Modbus
void mqtt_to_modbus_handler(const char *topic, const char *data) {
cJSON *cmd = cJSON_Parse(data);
if (!cmd) return;
char *device_name = cJSON_GetObjectItem(cmd, "device")->valuestring;
cJSON *params = cJSON_GetObjectItem(cmd, "params");
// 查找设备
for (int i = 0; i < modbus_device_count; i++) {
if (strcmp(modbus_devices[i].device_name, device_name) == 0) {
uint16_t addr = cJSON_GetObjectItem(params, "address")->valueint;
uint16_t value = cJSON_GetObjectItem(params, "value")->valueint;
modbus_write_holding(modbus_devices[i].unit_id, addr, value);
break;
}
}
cJSON_Delete(cmd);
}七、路由引擎与消息分发
7.1 路由规则
python
class RoutingEngine:
def __init__(self, config: dict):
self.routes = config.get("routes", [])
self.destinations = {} # 目标系统注册
def register_destination(self, name: str, sender):
"""注册目标系统"""
self.destinations[name] = sender
def route(self, message: InternalMessage):
"""根据路由规则转发消息"""
for route in self.routes:
if self._match_route(message, route):
destination = route["destination"]
if destination in self.destinations:
sender = self.destinations[destination]
asyncio.create_task(
sender.send(message, route.get("options", {}))
)
def _match_route(self, message: InternalMessage, route: dict) -> bool:
"""检查消息是否匹配路由规则"""
conditions = route.get("conditions", {})
for key, expected in conditions.items():
if key == "protocol":
if message.source_protocol != expected:
return False
elif key == "device_id":
if message.source_device_id != expected:
return False
elif key == "msg_type":
if message.msg_type != expected:
return False
elif key == "data_key":
if key not in message.data:
return False
return True路由规则配置示例：
json
{
"routes": [
{
"conditions": { "protocol": "modbus_tcp", "msg_type": "telemetry" },
"destination": "iot_platform",
"options": { "qos": 1 }
},
{
"conditions": { "protocol": "mqtt", "device_id": "alarm_*" },
"destination": "alert_system",
"options": { "priority": "high" }
},
{
"conditions": { "msg_type": "telemetry" },
"destination": "database",
"options": { "batch_size": 100 }
}
]
}八、连接管理与故障恢复
8.1 设备连接状态管理
python
class ConnectionManager:
def __init__(self):
self.devices: Dict[str, DeviceStatus] = {}
self.heartbeat_timeout = 60 # 60秒无心跳判定离线
def update_heartbeat(self, device_id: str, protocol: str):
"""更新设备心跳"""
if device_id not in self.devices:
self.devices[device_id] = DeviceStatus(
device_id=device_id,
protocol=protocol,
online=True,
last_seen=time.time()
)
else:
self.devices[device_id].last_seen = time.time()
self.devices[device_id].online = True
def check_timeouts(self):
"""检查设备超时"""
now = time.time()
for device_id, status in self.devices.items():
if status.online and (now - status.last_seen) > self.heartbeat_timeout:
status.online = False
logger.warning(f"Device offline: {device_id}")
# 触发离线回调
self._on_device_offline(device_id)
def _on_device_offline(self, device_id: str):
"""设备离线处理"""
# 通知应用层
# 清理资源
# 记录日志
pass九、性能优化
9.1 并发处理
python
import asyncio
from concurrent.futures import ThreadPoolExecutor
class GatewayEngine:
def __init__(self, config: dict):
self.adapters: Dict[str, ProtocolAdapter] = {}
self.transformer = DataTransformer(config)
self.router = RoutingEngine(config)
self.connection_mgr = ConnectionManager()
self.executor = ThreadPoolExecutor(max_workers=10)
async def start(self):
"""启动网关引擎"""
# 启动所有适配器
for name, adapter in self.adapters.items():
adapter.on_message(self._on_message)
await adapter.start()
# 启动连接检查
asyncio.create_task(self._health_check_loop())
def _on_message(self, message: InternalMessage):
"""消息处理管道"""
# 1. 数据转换
message = self.transformer.transform(message)
# 2. 更新心跳
self.connection_mgr.update_heartbeat(
message.source_device_id,
message.source_protocol
)
# 3. 路由分发
self.router.route(message)
async def _health_check_loop(self):
"""健康检查循环"""
while True:
self.connection_mgr.check_timeouts()
await asyncio.sleep(10)十、总结
协议网关是物联网系统中连接异构设备的核心枢纽。通过沧州虎王科技的设计与实践，我们实现了：
- 统一接入：Modbus、MQTT、HTTP等多种协议通过适配器模式统一接入
- 数据标准化：所有协议数据转换为统一的内部消息格式
- 灵活路由：基于规则的消息路由引擎，支持多目标分发
- 高可用：连接管理、故障恢复和断线重连机制保障系统可靠性
ESP32工具箱V2.0在网关部署阶段提供了串口调试和GPIO测试功能，特别适合RS485/Modbus线路的现场调试。物联网平台已内置协议网关服务，支持拖拽式配置适配器和路由规则，大幅降低了集成复杂度。
作者 ：沧州虎王科技技术团队
标签 ：物联网、嵌入式、ESP32
产品推荐：ESP32工具箱V2.0 | 随身WiFi硬件调试工具(hardware.czkree.com) | 物联网平台