python3 使用canal监听后，kafka数据同步从mysql到clickhouse数据库 完整复盘
#前置条件
1、您已经部署了mysql，并打开了bin-log日志，部署了clickhouse，与对应的库表
2、已经部署了kafka，zookeeper
3、已经部署了cannal ，并可以正常将数据写入到kafka中。
项目目录结构 (Project Directory Structure)
mysql_to_clickhouse/
├── config.yaml # 配置文件 (Configuration file)
├── daemon.py # 守护进程脚本 (Daemon process script)
├── daemon.pyc # 守护进程编译文件 (Daemon process compiled file)
└── mysql_to_clickhouse.py # 主程序文件 (Main program file)文件说明 (File Description)
| 文件 (File) | 类型 (Type) | 说明 (Description) |
|---|---|---|
config.yaml |
配置文件 | 存放项目配置参数 (Stores project configuration parameters) |
daemon.py |
Python 脚本 | 守护进程相关逻辑 (Daemon process related logic) |
daemon.pyc |
Python 编译文件 | daemon.py 的编译产物 (Compiled output of daemon.py) |
mysql_to_clickhouse.py |
Python 脚本 | MySQL 转 clickhouse 的主程序 (Main program for MySQL to clickhouse conversion) |
下面一一来介绍文件内容：
config.yaml
bash
#kafka连接信息
kafka:
#填写主机名
host: '127.0.0.1:9092'
brokers: 'message:9092,web:9092'
#获取数据库表操作的topic
database_topic: canal_example
error_topic: canal_example
#kafka消费偏移: smallest - 从最早消费; largest - 从最新消费
offset_reset: largest
auto_commit : False
# leiting
clickhouse:
http_api: http://192.168.1.100:8123
host:
- ['192.168.1.100',9000]
- ['192.168.1.100',9000]
username: default
password: shuxiaYuesee@72442w
database: DEISPssfei
#集群名称
cluster: shard_cluster
# canal -> ClickHouse 同步配置
sync:
# canal 推送里的源库名(msg.database),不匹配则丢弃
source_database: DEISPssfei
# 6 张待同步表的白名单
tables:
- cxx_user
- cxx_user_info
- cxx_kings
# 每张表 MySQL 主键列名(用于 UPDATE/DELETE 时定位行);
# 不在表里的回退到 canal payload.pkNames[0],再兜底为 'id'。
pk:
cxx_user: id
cxx_user_info: id
cxx_kings: id
# MySQL 零时间归一化值,避免 CK DateTime 解析失败
zero_datetime: '1970-01-01 00:00:00'
# MySQL 类型 -> CK 类型映射(更新时按需扩展)
ck_type_map:
'int(11)': Int32
'int(10) unsigned': UInt32
'bigint(20)': Int64
'bigint(20) unsigned': UInt64
'tinyint(1)': UInt8
'tinyint(4)': Int8
'smallint(6)': Int16
'varchar(256)': String
'varchar(128)': String
'varchar(64)': String
'varchar(32)': String
'datetime': DateTime
'date': Date
'decimal(10,2)': Decimal(10,2)
# CK 端列类型与 MySQL 不一致时的覆盖(优先于 ck_type_map 推导)。
# 例:MySQL depart_id/police_uid 是数值,CK 建表却是 String,
# 不覆盖会生成不带引号的裸数字,写 String 列报类型错误。
ck_column_types:
cxx_kings:
depart_id: String
police_uid: String
# ClickHouse 排序键(ORDER BY ...)。ClickHouse 不允许 ALTER TABLE UPDATE
# 排序键列,所以这些列从 UPDATE SET 中跳过,只更新非键列。
# 来源:jianbiao.txt 里每张表的 ORDER BY (...) 子句。
sort_keys:
cxx_user: [call_time, alarm_no, id]
cxx_user_info: [alarm_no, id]
cxx_kings: [alarm_no, id]
# ClickHouse 分区键列(PARTITION BY toYYYYMM(x) 里的 x)。
# 分区键列同样不允许 ALTER TABLE UPDATE,需要与 sort_keys 一起跳过。
partition_keys:
cxx_user: [call_time]
cxx_user_info: [created_at]
cxx_kings: [created_at]
# 每张 CK 本地表的列顺序(与 jianbiao.txt 的 CREATE TABLE 保持一致);
# INSERT 时按此顺序生成列名列表,确保与建表脚本可读一致。
# canal 推送里没有的列(CK 端有 DEFAULT 表达式的派生列)会被跳过,
# 由 CK 在写入时按 DEFAULT 自动填充。
# 顺序说明:id 在最前(便于对齐主键),info_id 紧跟其后(CK 去重主键);
# 其它列保持建表脚本顺序。
column_orders:
cxx_user:
- id
- info_id
- region
- alarm_no
- depart_id
- person_no
- person_name
- in_call_time
- call_time
- end_call_time
- call_time_long
- caller_phone
- caller_name
- caller_sex
- happen_place
- content
- manage_depart
- class_code
- type_code
- subclass_code
- msg_code
- level
- bill_status
- cycs
- lat
- lng
- phone_number
- caller_idcard
- created_at
- updated_at
- status
- degree
- is_repeat
- attention_uid
cxx_user_info:
- id
- info_id
- region
- alarm_no
- process_no
- depart_id
- person_no
- person_name
- platform_no
- platform_ip
- process_time
- process_depart_id
- accpet_person_no
- accept_person_name
- over_time
- send_order_time
- send_over_time
- alarm_status
- created_at
- updated_at
cxx_kings:
- id
- info_id
- region
- alarm_no
- process_no
- feedback_no
- fk_time
- fk_time_long
- input_depart_id
- platform_no
- platform_ip
- feedback_person_no
- feedback_person_name
- command_name
- depart_id
- go_name
- go_time
- go_arrive_time
- class_code
- type_code
- subclass_code
- handle_code
- is_whether
- happen_time
- end_time
- address_class_code
- address_type_code
- desction
- created_at
- updated_at
python
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import atexit
from signal import SIGTERM ,SIGKILL
import os,sys,time,subprocess
class Daemon:
"""
A generic daemon class.
Usage: subclass the Daemon class and override the run() method
"""
def __init__(self, pidfile, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
self.stdin = stdin
#self.stdout = stdout
self.stderr = stderr
self.pidfile = pidfile
def daemonize(self):
"""
do the UNIX double-fork magic, see Stevens' "Advanced
Programming in the UNIX Environment" for details (ISBN 0201563177)
http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
"""
try:
pid = os.fork()
if pid > 0:
# exit first parent
sys.exit(0)
except OSError as e:
sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# decouple from parent environment
os.chdir(os.path.dirname(os.path.abspath(__file__)))
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError as e:
sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = open(self.stdin, 'r')
#so = file(self.stdout, 'a+')
se = open(self.stderr, 'ab+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
#os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
open(self.pidfile,'w+').write("%s\n" % pid)
def delpid(self):
os.remove(self.pidfile)
def kill_child_processes(self,parent_pid, sig=SIGKILL):
try:
try:
pgid = os.getpgid(int(parent_pid))
#print 'killpg',pgid
os.killpg(pgid,sig)
except Exception as e:
ps_command = subprocess.Popen("ps -o pid --ppid %d --noheaders" % parent_pid, shell=True, stdout=subprocess.PIPE)
ps_output = ps_command.stdout.read()
retcode = ps_command.wait()
assert retcode == 0, "ps command returned %d" % retcode
for pid_str in ps_output.split("\n")[:-1]:
os.kill(int(pid_str), sig)
except AssertionError:
return
except Exception as e:
return
def check_pid(self,pid):
""" Check For the existence of a unix pid. """
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def start(self):
"""
Start the daemon
"""
# Check for a pidfile to see if the daemon already runs
try:
pf = open(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if pid and self.check_pid(pid):
message = "pidfile %s already exist. Daemon already running?\n"
sys.stderr.write(message % self.pidfile)
sys.exit(1)
# Start the daemon
self.daemonize()
self.run()
def stop(self):
"""
Stop the daemon
"""
# Get the pid from the pidfile
try:
pf = open(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if not pid:
message = "pidfile %s does not exist. Daemon not running?\n"
sys.stderr.write(message % self.pidfile)
return # not an error in a restart
# Try killing the daemon process
try:
while 1:
self.kill_child_processes(pid, SIGTERM)
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError as err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
print(str(err))
sys.exit(1)
def restart(self):
"""
Restart the daemon
"""
self.stop()
self.start()
def run(self):
"""
You should override this method when you subclass Daemon. It will be called after the process has been
daemonized by start() or restart().
"""mysql_to_clickhouse.py
python
#!/usr/bin/python
#-*- coding: utf-8 -*-
#用于同步mysql到clickhouse数据库
import math
import redis
import pymysql
#import queue as Queue
#import Queue
import yaml,logging,json
from logging.handlers import TimedRotatingFileHandler,RotatingFileHandler
import requests
from datetime import datetime,date,timedelta
import time,timeit,sys,os,threading
from confluent_kafka import Consumer
from confluent_kafka import Producer
from confluent_kafka import KafkaError
from daemon import Daemon
import uuid
def generate_info_id():
# 生成 UUID 字面字符串(8-4-4-4-12 形式),写入 CK 时配合
# UUIDStringToNum() 转成 16 字节二进制后存入 info_id(FixedString(16))。
#
# 与 PHP 端 SyncRiskAssessPoliceUidUpdateProcess 等效:
# UUIDStringToNum('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx') -> 16 bytes
#
# canal MySQL 源表不存 info_id,首次 INSERT 时由本方法生成;
# 之后 UPDATE/DELETE 一律不再修改 info_id(详见 _build_alter_update_sql)。
return str(uuid.uuid4())
class MyDaemon(Daemon):
def api_post(self, sql, timeout=10):
base_url = self.http_api.rstrip('/')
url = base_url + '/'
auth = (self.username, self.password) if self.username else None
headers = {'Content-Type': 'text/plain; charset=utf-8'}
r = requests.post(
url,
params={'database': self.database},
data=sql.encode('utf-8'),
headers=headers,
auth=auth,
timeout=timeout
)
if r.status_code != 200:
logging.error('调用clickhouseAPI异常: %s', r.text[:500])
raise RuntimeError('ck http {}: {}'.format(r.status_code, r.text[:500]))
return r.text
def _ck_value(self, value, mysql_type, ck_type=None):
# null 兜底:字符串类 -> '',数字类 -> 0,日期类 -> zero_dt。
# ClickHouse 不接受 NULL 的字符串/整数字段(取决于 DEFAULT),直接写
# 空字符串/0 既符合 CK 端 DEFAULT 约定,也避免下游 JOIN 时类型异常。
if value is None or value == '' or value == 'NULL' or value == 'null':
if ck_type:
ck_t = ck_type.lower()
if any(k in ck_t for k in ('string', 'fixedstring')):
return "''"
if any(k in ck_t for k in ('int', 'uint', 'float', 'decimal')):
return '0'
if 'datetime' in ck_t:
return "'{}'".format(self.zero_dt)
if ck_t.startswith('date'):
return "'1970-01-01'"
return "''"
# CK 端列类型优先:MySQL 是数值但 CK 建表是 String 时(如
# cdp_process_police_alarm_delete.depart_id),必须按 String 加引号。
if ck_type:
ck_t = ck_type.lower()
if any(k in ck_t for k in ('string', 'fixedstring')):
return "'{}'".format(str(value).replace("'", "\\'"))
t = (mysql_type or '').lower()
# datetime:空或 0000-00-00 00:00:00 -> zero_datetime
if t.startswith('datetime'):
if value.startswith('0000-') or value == '0000-00-00 00:00:00':
value = self.zero_dt
return "'{}'".format(str(value).replace("'", "\\'"))
if t.startswith('date'):
if value.startswith('0000-'):
value = '1970-01-01'
return "'{}'".format(str(value).replace("'", "\\'"))
if t.startswith('int') or t.startswith('bigint') or t.startswith('tinyint') or t.startswith('smallint'):
try:
# 直接转 int,避免 float 在 >2^53 时丢精度
s = str(value).strip().replace(',', '')
return str(int(s))
except Exception:
return '0'
if t.startswith('decimal'):
try:
return str(float(value))
except Exception:
return '0'
return "'{}'".format(str(value).replace("'", "\\'"))
def ck_type_for(self, table, col, mysql_type):
# CK 列类型:先看配置里的显式覆盖,再按 MySQL 类型映射推导
override = (self.ck_column_types.get(table) or {}).get(col)
if override:
return override
return self.ck_type_map.get((mysql_type or '').lower())
def _build_insert_sql(self, table, data, mysql_type_map):
# canal 推送的单行字典(str -> str)
info_id = generate_info_id()
pk_col = self.pk_for_table(table)
# 优先按配置中的列顺序生成 INSERT,与建表脚本保持一致;
# canal 推送里没有的列(CK 端 DEFAULT 派生列等)会被跳过,
# 由 CK 在写入时按 DEFAULT 表达式填充。
ordered_cols = self.column_orders.get(table)
if ordered_cols:
cols = list(ordered_cols)
else:
cols = [pk_col, 'info_id'] + [c for c in data.keys()
if c != pk_col and c != 'info_id']
# canal 通常把 PK 推进 data,但部分版本不会;
# 保证 PK 列一定出现在 INSERT,否则 CK 会因缺主键报错。
if pk_col not in cols:
cols = [pk_col] + cols
cols_sql = []
vals = []
for c in cols:
if c == pk_col:
cols_sql.append('`{}`'.format(pk_col))
vals.append(self._ck_value(data.get(pk_col, '0'),
mysql_type_map.get(pk_col, 'int(11)'),
self.ck_type_for(table, pk_col, mysql_type_map.get(pk_col))))
elif c == 'info_id':
# 与 PHP 端 UUIDStringToNum(...) 等效
cols_sql.append('`info_id`')
vals.append("UUIDStringToNum('{}')".format(info_id))
else:
if c not in data:
# canal 未推送的列(CK DEFAULT 派生列),让 CK 自动填充
continue
cols_sql.append('`{}`'.format(c))
vals.append(self._ck_value(data.get(c),
mysql_type_map.get(c, 'varchar(256)'),
self.ck_type_for(table, c, mysql_type_map.get(c))))
values_sql = '({})'.format(', '.join(vals))
table_all = '{}_all'.format(table)
sql = "INSERT INTO {} ({}) VALUES {}".format(table_all, ', '.join(cols_sql), values_sql)
return sql
def _build_alter_update_sql(self, table, after, mysql_type_map, pk_name):
pk_value = after.get(pk_name)
if not pk_value:
raise ValueError('update event missing pk {} for table {}'.format(pk_name, table))
# ClickHouse 排序键(ORDER BY ...)和分区键(PARTITION BY ...)都不可
# UPDATE,需要一起跳过;否则报 Cannot UPDATE key column。
skip_cols = set(self.sort_keys.get(table, []))
skip_cols |= set(self.partition_keys.get(table, []))
# date 是 CK 端的 DEFAULT 派生列(toDate(分区列)),canal 不推,也不更新
skip_cols.add('date')
# 全量 SET:按主键定位行后,把所有非 PK、非 info_id、非排序键的列
# 直接写成 after 里的新值。不做 diff 比对 ------ canal 推的 UPDATE
# 本就是「按 PK 把整行刷新」的语义,无意义 SET 也只是写等值,代价可忽略。
# info_id 在首次 INSERT 时生成,UPDATE 时不动,避免破坏 CK 端的去重主键。
set_parts = []
for col in after.keys():
if col == pk_name or col == 'info_id' or col in skip_cols:
continue
ck_v = self._ck_value(after.get(col),
mysql_type_map.get(col, 'varchar(256)'),
self.ck_type_for(table, col, mysql_type_map.get(col)))
set_parts.append('`{}` = {}'.format(col, ck_v))
if not set_parts:
logging.warning('update event has no settable cols, skip. table=%s id=%s',
table, pk_value)
return None
pk_ck = self._ck_value(pk_value, mysql_type_map.get(pk_name, 'int(11)'),
self.ck_type_for(table, pk_name, mysql_type_map.get(pk_name)))
sql = ("ALTER TABLE `{}` ON CLUSTER {} UPDATE {} "
"WHERE `{}` = {}").format(table, self.cluster, ', '.join(set_parts),
pk_name, pk_ck)
return sql
def _build_alter_delete_sql(self, table, data, mysql_type_map, pk_name):
pk_value = data.get(pk_name)
if not pk_value:
raise ValueError('delete event missing pk {} for table {}'.format(pk_name, table))
pk_ck = self._ck_value(pk_value, mysql_type_map.get(pk_name, 'int(11)'),
self.ck_type_for(table, pk_name, mysql_type_map.get(pk_name)))
sql = ("ALTER TABLE `{}` ON CLUSTER {} DELETE "
"WHERE `{}` = {}").format(table, self.cluster, pk_name, pk_ck)
return sql
def _delivery_callback(self, err, msg):
"""producer callback (poll or flush)"""
if err:
logging.error('Message failed delivery: %s' % err)
def _exec_write(self, sql ,msg ):
try:
ret = self.api_post(sql)
body = (ret or '').strip()
# CK HTTP 成功时 body 是 "table\trows\n";只在出现异常关键字时升级日志
if body and any(k in body.lower() for k in ('exception', 'error', 'failed')):
logging.warning('ck write resp: %s | sql: %s', body, sql[:200])
else:
logging.info('ok')
except Exception as e:
logging.exception('ck write failed. sql=%s err=%s', sql, e)
logging.info('重新回推到kafka')
payload = json.dumps(msg,ensure_ascii=False).encode('utf-8')
self.producer.produce(self.config['kafka']['error_topic'],payload,callback=self._delivery_callback,)
self.producer.poll(0)
remaing = self.producer.flush(timeout=5)
if remaing > 0 :
logging.error("kafka flush timeout")
def _parse_pk_name(self,pk_names):
return pk_names[0] if pk_names else None
"""取表的主键列名:配置优先,否则 canal 推的 pkNames,否则 'id'"""
def pk_for_table(self, table, fallback_pk=None):
return self.pk_map.get(table) or fallback_pk or 'id'
def handle(self, msg):
database = msg.get('database', '')
table = msg.get('table', '')
event_type = msg.get('type', '')
mysql_type_map = msg.get('mysqlType', {}) or {}
pk_names = msg.get('pkNames', []) or []
data_list = msg.get('data', []) or []
if database != self.source_database:
logging.error('数据库未定义')
return False
if table not in self.sync_tables:
logging.error('同步表未定义')
return False
if not data_list:
logging.error('kafka数据没有data自定义数据')
return False
pk_name = self.pk_for_table(table, fallback_pk=self._parse_pk_name(pk_names))
for row in data_list:
try:
if event_type == 'INSERT':
sql = self._build_insert_sql(table, row, mysql_type_map)
self._exec_write(sql,msg)
elif event_type == 'UPDATE':
# canal: 'data' 是新值字典;不再读 'old',直接全量 SET
sql = self._build_alter_update_sql(table, row, mysql_type_map, pk_name)
if sql:
self._exec_write(sql,msg)
elif event_type == 'DELETE':
sql = self._build_alter_delete_sql(table, row, mysql_type_map, pk_name)
self._exec_write(sql,msg)
else:
logging.warning('unknown event type: %s', event_type)
except Exception as e:
logging.exception('handle row failed. table=%s type=%s err=%s',
table, event_type, e)
def get_kafka_message(self):
consumer = None
# 默认改手动 commit,处理成功才提交,避免重启时重复消费
enable_auto_commit = bool(self.config['kafka'].get('auto_commit', False))
consumer_conf = {
'bootstrap.servers': self.config['kafka']['host'],
'group.id': 'sync_ck_20260808133600',#当前脚本的名称_日期精确到秒
'client.id': 'sync_ck_20260808133600',#当前脚本的名称_日期精确到秒
'auto.offset.reset': self.config['kafka']['offset_reset'],
'enable.auto.commit': enable_auto_commit
}
consumer = Consumer(**consumer_conf)
def print_assignment(consumer, partitions):
logging.info("Assignment: {}".format(partitions))
consumer.subscribe([self.config['kafka']['database_topic']], on_assign=print_assignment)
while 1:
try:
#每次消费10条数据
messages = consumer.consume(num_messages=1,timeout=1.0)
#无数据或者超时可能返回空
if not messages:
time.sleep(0.01)
continue
for message in messages:
partition = message.partition()
offset = message.offset()
#检验Message对象
if message.error():
if message.error().code() == KafkaError._PARTITION_EOF:
logging.info('partition: %d reached end at offset %d.', partition, offset)
pass
else:
logging.error("kafka consumer error! {}".format(message.error()))
continue
row = message.value()
if row is not None:
try :
result = json.loads(row.decode('utf-8'))
#logging.info(json.dumps(result,ensure_ascii=False)) #转成json 中文不转义
except Exception as e:
logging.error('kafka数据处理失败: %s', str(e))
continue
if result:
#处理sql
logging.info(result)
self.handle(result)
except KeyboardInterrupt:
logging.error('Ctrl+C,终止运行')
#停止消费，提交偏移量
consumer.close()
return
except Exception as e:
logging.exception('连接kafka时错误: %s', str(e))
time.sleep(1)
logging.warning('consumer thread exited')
def run(self):
with open(os.path.dirname(os.path.abspath(__file__)) + '/config.yaml') as config_file:
self.config = yaml.safe_load(config_file)
name = 'mysql_to_clickhouse'
logging.basicConfig(level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
handler = RotatingFileHandler('/var/log/%s.log' % name, maxBytes=134217728, backupCount=7)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logging.getLogger('').addHandler(handler)
logging.info('启动 [%s]', name)
self.db = None
self.queen_size = 500
self.http_api = self.config['clickhouse']['http_api'].rstrip('/') #clickhouse数据库api
self.username = self.config['clickhouse'].get('username', 'default') #数据库用户名
self.password = self.config['clickhouse']['password']
self.database = self.config['clickhouse']['database']
self.cluster = self.config['clickhouse']['cluster']
sync_cfg = self.config.get('sync', {})
self.sync_tables = set(sync_cfg.get('tables', []))
self.source_database = sync_cfg.get('source_database', 'DEISPssfei')
self.zero_dt = sync_cfg.get('zero_datetime', '1970-01-01 00:00:00')
self.column_orders = sync_cfg.get('column_orders', {})
# 各表 MySQL 主键列名;UPDATE/DELETE 用它定位行,INSERT 用它选 PK 列
self.pk_map = sync_cfg.get('pk', {})
# MySQL 类型 -> CK 类型映射;null 兜底时按 CK 类型决定默认值
self.ck_type_map = sync_cfg.get('ck_type_map', {})
# CK 端列类型覆盖(MySQL 与 CK 类型不一致的列)
self.ck_column_types = sync_cfg.get('ck_column_types', {})
# ClickHouse 排序键(ORDER BY ...),UPDATE 时跳过这些列
self.sort_keys = sync_cfg.get('sort_keys', {})
# ClickHouse 分区键列(PARTITION BY ...),UPDATE 时同样跳过
self.partition_keys = sync_cfg.get('partition_keys', {})
self.producer = Producer ({
'bootstrap.servers': self.config['kafka']['brokers']
})
try:
#监控相似度参数
t = threading.Thread(target=self.get_kafka_message)
t.setDaemon(True)
t.start()
logging.warning('进行Mysql到clickhouse的同步...')
while 1:
try:
time.sleep(600)
logging.info('running')
except KeyboardInterrupt:
logging.error('Ctrl+C,终止运行')
sys.exit(1)
except Exception as e:
logging.exception('连接kafka时错误: %s', str(e))
time.sleep(1)
except Exception as e:
logging.exception('同步Mysql数据到clickhouse异常: %s', str(e))
sys.exit(1)
if __name__ == "__main__":
daemon = MyDaemon('/var/run/mysql_to_clickhouse.pid')
#daemon.run()
#sys.exit(1)
if len(sys.argv) == 2:
if 'start' == sys.argv[1]:
daemon.start()
elif 'stop' == sys.argv[1]:
daemon.stop()
elif 'restart' == sys.argv[1]:
daemon.restart()
else:
print("Unknown command")
sys.exit(2)
sys.exit(0)
else:
print("usage: %s start|stop|restart" % sys.argv[0])
sys.exit(2)运行：python3 mysql_to_lightningdb.py start
日志：tailf /var/log/mysql_to_lightningdb.log