# ===================== StreamLoad函数 =====================
def json_serial(obj):
if isinstance(obj, (datetime, date)):
return obj.strftime("%Y-%m-%d %H:%M:%S")
raise TypeError(f"Object of type {type(obj).name} is not JSON serializable")
def doris_stream_load(data_rows: List[list]) -> bool:
"""Doris StreamLoad写入明细表ods_jice_detail_data"""
#print(f"【StreamLoad实际接收行数】{len(data_rows)}", flush=True)
if not data_rows:
return True
url = f"http://{doris_load_host}/api/{target_doris_db}/{target_doris_table}/_stream_load"
headers = {
"Expect": "100-continue",
"format": "json",
"read_json_by_line": "true",
"Content-Type": "application/json",
"strip_outer_array": "false",
"label": f"test-{uuid.uuid4()}"
}
auth = (doris_user, doris_pwd)
body = "\n".join([json.dumps(line, ensure_ascii=False, default=json_serial) for line in data_rows])
try:
resp = requests.put(url, headers=headers, auth=auth, data=body.encode("utf-8"), timeout=120, allow_redirects=False)
#print(f"【StreamLoad http状态码】{resp.status_code}", flush=True)
# 处理FE 307重定向,携带完整header请求BE
if resp.status_code == 307:
redirect_url = resp.headers.get("Location")
#print(f"【检测到307重定向,跳转地址:{redirect_url}】", flush=True)
resp = requests.put(redirect_url, headers=headers, auth=auth, data=body.encode("utf-8"), timeout=120)
#print(f"【重定向后http状态码】{resp.status_code}", flush=True)
headers["label"] = f"jice-{uuid.uuid4()}"
resp_text = resp.text.strip()
#print(f"【Doris原始返回内容】{resp_text}", flush=True)
if not resp_text:
print(f"【StreamLoad严重失败】Doris返回空响应!批次行数:{len(data_rows)}", flush=True)
print(f"响应头信息:{resp.headers}", flush=True)
return False
try:
res_json = json.loads(resp_text)
except json.JSONDecodeError as je:
print(f"【响应非合法JSON】原始返回内容:{resp_text}", flush=True)
print(f"JSON解析异常:{str(je)}", flush=True)
traceback.print_exc()
return False
#print(f"【Doris返回完整信息】{res_json}", flush=True)
status = res_json.get("Status")
load_rows = res_json.get("NumberLoadedRows", 0)
#print(f"【导入统计】请求行数:{len(data_rows)}, Doris实际加载行数:{load_rows}", flush=True)
if status == "Success":
print(f"【StreamLoad成功】LoadId:{res_json.get('LoadId')}", flush=True)
return True
else:
# 这里包含 Publish Timeout、ETL失败、数据格式错误等隐性失败
#print(f"【StreamLoad业务失败】状态:{status},原因:{res_json.get('Message')}", flush=True)
return False
except Exception as e:
print(f"【StreamLoad网络异常】异常信息:{str(e)}", flush=True)
traceback.print_exc()
return False