上一章节说到DorisWriter,这回说一下里面最核心的,真正干活的人---DorisStreamLoad
3.DorisStreamLoad--核心
kotlin
public class DorisStreamLoad implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoad.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final LabelGenerator labelGenerator;
private final byte[] lineDelimiter;
private static final String LOAD_URL_PATTERN = "http://%s/api/%s/%s/_stream_load";
private static final String ABORT_URL_PATTERN = "http://%s/api/%s/_stream_load_2pc";
public static final String JOB_EXIST_FINISHED = "FINISHED";
private String loadUrlStr;
private String hostPort;
private String abortUrlStr;
private final String user;
private final String passwd;
private final String db;
private final String table;
private final boolean enable2PC;
private final boolean enableDelete;
private final Properties streamLoadProp;
private final RecordStream recordStream;
private volatile Future<RespContent> pendingLoadFuture;
private volatile Exception httpException = null;
private final CloseableHttpClient httpClient;
private final ExecutorService executorService;
private boolean loadBatchFirstRecord;
private volatile String currentLabel;
private boolean enableGroupCommit;
private boolean enableGzCompress;
public DorisStreamLoad(String hostPort, DorisOptions dorisOptions, DorisExecutionOptions executionOptions, LabelGenerator labelGenerator, CloseableHttpClient httpClient) {
this.hostPort = hostPort;
String[] tableInfo = dorisOptions.getTableIdentifier().split("\.");
this.db = tableInfo[0];
this.table = tableInfo[1];
this.user = dorisOptions.getUsername();
this.passwd = dorisOptions.getPassword();
this.labelGenerator = labelGenerator;
this.loadUrlStr = String.format("http://%s/api/%s/%s/_stream_load", hostPort, this.db, this.table);
this.abortUrlStr = String.format("http://%s/api/%s/_stream_load_2pc", hostPort, this.db);
this.enable2PC = executionOptions.enabled2PC();
this.streamLoadProp = executionOptions.getStreamLoadProp();
this.enableDelete = executionOptions.getDeletable();
this.httpClient = httpClient;
String threadName = String.format("stream-load-upload-%s-%s", labelGenerator.getSubtaskId(), labelGenerator.getTableIdentifier());
this.executorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), new ExecutorThreadFactory(threadName));
this.recordStream = new RecordStream(executionOptions.getBufferSize(), executionOptions.getBufferCount(), executionOptions.isUseCache());
if (this.streamLoadProp.getProperty("format", "csv").equals("arrow")) {
this.lineDelimiter = null;
} else {
this.lineDelimiter = EscapeHandler.escapeString(this.streamLoadProp.getProperty("line_delimiter", "\n")).getBytes();
}
this.enableGroupCommit = this.streamLoadProp.containsKey("group_commit") && !this.streamLoadProp.getProperty("group_commit").equalsIgnoreCase("off_mode");
this.enableGzCompress = this.streamLoadProp.getProperty("compress_type", "").equals("gz");
this.loadBatchFirstRecord = true;
}
public void setHostPort(String hostPort) {
this.hostPort = hostPort;
this.loadUrlStr = String.format("http://%s/api/%s/%s/_stream_load", hostPort, this.db, this.table);
this.abortUrlStr = String.format("http://%s/api/%s/_stream_load_2pc", hostPort, this.db);
}
public Future<RespContent> getPendingLoadFuture() {
return this.pendingLoadFuture;
}
// 预提交中断
public void abortPreCommit(String labelPrefix, long chkID) throws Exception {
long startChkID = chkID;
LOG.info("abort for labelPrefix {}, concat labelPrefix {}, start chkId {}.", new Object[]{labelPrefix, this.labelGenerator.getConcatLabelPrefix(), chkID});
while(true) {
try {
String label = this.labelGenerator.generateTableLabel(startChkID);
LOG.info("start a check label {} to load.", label);
HttpPutBuilder builder = new HttpPutBuilder();
builder.setUrl(this.loadUrlStr).baseAuth(this.user, this.passwd).addCommonHeader().enable2PC().setLabel(label).setEmptyEntity().addProperties(this.streamLoadProp);
RespContent respContent = this.handlePreCommitResponse(this.httpClient.execute(builder.build()));
Preconditions.checkState("true".equals(respContent.getTwoPhaseCommit()));
if ("Label Already Exists".equals(respContent.getStatus())) {
if ("FINISHED".equals(respContent.getExistingJobStatus())) {
throw new DorisException("Load status is Label Already Exists and load job finished, change you label prefix or restore from latest savepoint!");
}
Matcher matcher = ResponseUtil.LABEL_EXIST_PATTERN.matcher(respContent.getMessage());
if (matcher.find()) {
Preconditions.checkState(label.equals(matcher.group(1)));
long txnId = Long.parseLong(matcher.group(2));
LOG.info("abort {} for exist label {}", txnId, label);
this.abortTransaction(txnId);
++startChkID;
continue;
}
LOG.error("response: {}", respContent.toString());
throw new DorisException("Load Status is Label Already Exists, but no txnID associated with it!");
}
LOG.info("abort {} for check label {}.", respContent.getTxnId(), label);
this.abortTransaction(respContent.getTxnId());
} catch (Exception var12) {
LOG.warn("failed to abort labelPrefix {}", labelPrefix, var12);
throw var12;
}
LOG.info("abort for labelPrefix {} finished", labelPrefix);
return;
}
}
// 写入数据,详情请看下面
public void writeRecord(byte[] record) throws InterruptedException {
this.checkLoadException();
try {
if (this.loadBatchFirstRecord) {
this.loadBatchFirstRecord = false;
} else if (this.lineDelimiter != null) {
this.recordStream.write(this.lineDelimiter);
}
this.recordStream.write(record);
} catch (InterruptedException var3) {
Thread.currentThread().interrupt();
if (this.httpException != null) {
throw new DorisRuntimeException(this.httpException.getMessage(), this.httpException);
} else {
LOG.info("write record interrupted, cause " + var3.getClass());
throw var3;
}
}
}
@VisibleForTesting
public RecordStream getRecordStream() {
return this.recordStream;
}
// 处理预提交请求
public RespContent handlePreCommitResponse(CloseableHttpResponse response) throws Exception {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 && response.getEntity() != null) {
String loadResult = EntityUtils.toString(response.getEntity());
LOG.info("load Result {}", loadResult);
RespContent respContent = (RespContent)OBJECT_MAPPER.readValue(loadResult, RespContent.class);
if (respContent != null && respContent.getLabel() != null && respContent.getTxnId() != null) {
return respContent;
} else {
throw new DorisRuntimeException("Response error : " + loadResult);
}
} else {
throw new StreamLoadException("stream load error: " + response.getStatusLine().toString());
}
}
// 停止加载数据
public RespContent stopLoad() throws InterruptedException {
try {
this.recordStream.endInput();
if (this.enableGroupCommit) {
LOG.info("table {} stream load stopped with group commit on host {}", this.table, this.hostPort);
} else {
LOG.info("table {} stream load stopped for {} on host {}", new Object[]{this.table, this.currentLabel, this.hostPort});
}
Preconditions.checkState(this.pendingLoadFuture != null);
return (RespContent)this.pendingLoadFuture.get();
} catch (InterruptedException var2) {
Thread.currentThread().interrupt();
if (this.httpException != null) {
throw new DorisRuntimeException(this.httpException.getMessage(), this.httpException);
} else {
throw var2;
}
} catch (ExecutionException var3) {
throw new DorisRuntimeException(var3);
}
}
// 开始加载数据
public void startLoad(String label, boolean isResume) throws IOException {
if (this.enableGroupCommit) {
label = null;
}
this.loadBatchFirstRecord = !isResume;
HttpPutBuilder putBuilder = new HttpPutBuilder();
this.recordStream.startInput(isResume);
if (this.enableGroupCommit) {
LOG.info("table {} stream load started with group commit on host {}", this.table, this.hostPort);
} else {
LOG.info("table {} stream load started for {} on host {}", new Object[]{this.table, label, this.hostPort});
}
this.currentLabel = label;
String executeMessage;
try {
InputStreamEntity entity = new InputStreamEntity(this.recordStream);
putBuilder.setUrl(this.loadUrlStr).baseAuth(this.user, this.passwd).addCommonHeader().addHiddenColumns(this.enableDelete).setLabel(label).setEntity(entity).addProperties(this.streamLoadProp);
if (this.enable2PC) {
putBuilder.enable2PC();
}
if (this.enableGzCompress) {
putBuilder.setEntity(new GzipCompressingEntity(entity));
}
if (this.enableGroupCommit) {
executeMessage = "table " + this.table + " start execute load with group commit";
} else {
executeMessage = "table " + this.table + " start execute load for label " + label;
}
Thread mainThread = Thread.currentThread();
this.pendingLoadFuture = this.executorService.submit(() -> {
LOG.info(executeMessage);
try {
CloseableHttpResponse execute = this.httpClient.execute(putBuilder.build());
RespContent respContent = this.handlePreCommitResponse(execute);
if (!LoadConstants.DORIS_SUCCESS_STATUS.contains(respContent.getStatus())) {
if (this.enable2PC && "Label Already Exists".equals(respContent.getStatus()) && !"FINISHED".equals(respContent.getExistingJobStatus())) {
LOG.info("try to abort {} cause status {}, exist job status {} ", new Object[]{respContent.getLabel(), respContent.getStatus(), respContent.getExistingJobStatus()});
this.abortLabelExistTransaction(respContent);
throw new LabelAlreadyExistsException("Exist label abort finished, retry");
} else {
String errMsg = String.format("table %s.%s stream load error: %s, see more in %s", this.getDb(), this.getTable(), respContent.getMessage(), respContent.getErrorURL());
LOG.error("Failed to load, {}", errMsg);
throw new DorisRuntimeException(errMsg);
}
} else {
return respContent;
}
} catch (NoRouteToHostException var7) {
LOG.error("Failed to connect, cause ", var7);
this.httpException = var7;
mainThread.interrupt();
throw new DorisRuntimeException("No Route to Host to " + this.hostPort + ", exception: " + var7);
} catch (Exception var8) {
LOG.error("Failed to execute load, cause ", var8);
this.httpException = var8;
mainThread.interrupt();
throw var8;
}
});
} catch (Exception var7) {
if (this.enableGroupCommit) {
executeMessage = "failed to stream load data with group commit";
} else {
executeMessage = "failed to stream load data with label: " + label;
}
LOG.warn(executeMessage, var7);
throw var7;
}
}
public void abortTransaction(long txnID) throws Exception {
HttpPutBuilder builder = new HttpPutBuilder();
builder.setUrl(this.abortUrlStr).baseAuth(this.user, this.passwd).addCommonHeader().addTxnId(txnID).setEmptyEntity().abort();
CloseableHttpResponse response = this.httpClient.execute(builder.build());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 && response.getEntity() != null) {
ObjectMapper mapper = new ObjectMapper();
String loadResult = EntityUtils.toString(response.getEntity());
LOG.info("abort Result {}", loadResult);
Map<String, String> res = (Map)mapper.readValue(loadResult, new TypeReference<HashMap<String, String>>() {
});
if (!"Success".equals(res.get("status"))) {
String msg = (String)res.get("msg");
if (msg != null && ResponseUtil.isAborted(msg)) {
LOG.info("transaction {} may have already been successfully aborted, skipping, abort response is {}", txnID, msg);
} else {
LOG.error("Fail to abort transaction. txnId: {}, error: {}", txnID, msg);
throw new DorisException("Fail to abort transaction, " + loadResult);
}
}
} else {
LOG.warn("abort transaction response: " + response.getStatusLine().toString());
throw new DorisRuntimeException("Fail to abort transaction " + txnID + " with url " + this.abortUrlStr);
}
}
public void abortTransactionByLabel(String label) throws Exception {
if (!StringUtils.isNullOrWhitespaceOnly(label)) {
HttpPutBuilder builder = new HttpPutBuilder();
builder.setUrl(this.abortUrlStr).baseAuth(this.user, this.passwd).addCommonHeader().setLabel(label).setEmptyEntity().abort();
CloseableHttpResponse response = this.httpClient.execute(builder.build());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 && response.getEntity() != null) {
ObjectMapper mapper = new ObjectMapper();
String loadResult = EntityUtils.toString(response.getEntity());
LOG.info("abort Result {}", loadResult);
Map<String, String> res = (Map)mapper.readValue(loadResult, new TypeReference<HashMap<String, String>>() {
});
if (!"Success".equals(res.get("status"))) {
String msg = (String)res.get("msg");
if (msg != null && ResponseUtil.isCommitted(msg)) {
throw new DorisException("try abort committed transaction by label, do you recover from old savepoint?");
} else {
LOG.error("Fail to abort transaction by label. label: {}, error: {}", label, msg);
throw new DorisException("Fail to abort transaction by label, " + loadResult);
}
}
} else {
LOG.warn("abort transaction by label response: " + response.getStatusLine().toString());
throw new DorisRuntimeException("Fail to abort transaction by label " + label + " with url " + this.abortUrlStr);
}
}
}
public void abortLabelExistTransaction(RespContent respContent) {
if (respContent != null && respContent.getMessage() != null) {
try {
Matcher matcher = ResponseUtil.LABEL_EXIST_PATTERN.matcher(respContent.getMessage());
if (matcher.find()) {
long txnId = Long.parseLong(matcher.group(2));
this.abortTransaction(txnId);
LOG.info("Finish to abort transaction {} for label already exist {}", txnId, respContent.getLabel());
}
} catch (Exception var5) {
LOG.error("Failed abort transaction {} for label already exist", respContent.getLabel());
}
}
}
}
(1) 写入流程源码分析
三个阶段:数据写入触发、缓冲区流转、发送HTTP请求
<1> 入口--DorisWriter和DorisStreamLoad
Flink 处理完一条数据后,通过 DorisWriter
调用 DorisStreamLoad.writeRecord
,将序列化后的字节数组 record
传入
kotlin
public void writeOneDorisRecord(DorisRecord record) throws IOException, InterruptedException {
if (record != null && record.getRow() != null) {
String tableKey = this.dorisOptions.getTableIdentifier();
if (record.getTableIdentifier() != null) {
tableKey = record.getTableIdentifier();
}
DorisStreamLoad streamLoader = this.getStreamLoader(tableKey);
if (!this.loadingMap.containsKey(tableKey)) {
LabelGenerator labelGenerator = this.getLabelGenerator(tableKey);
// 根据当前的checkpointID去生成对应的label标签,传给DorisStreamLoad对象,再由这个对象去发送http请求携带这个label标签
String currentLabel = labelGenerator.generateTableLabel(this.curCheckpointId);
// 开启加载数据
streamLoader.startLoad(currentLabel, false);
this.loadingMap.put(tableKey, true);
this.globalLoading = true;
this.registerMetrics(tableKey);
}
// 写入数据
streamLoader.writeRecord(record.getRow());
}
}
生成标签的LabelGenerator
类
kotlin
public String generateTableLabel(long chkId) {
Preconditions.checkState(this.tableIdentifier != null);
String label = String.format("%s_%s_%s_%s", this.labelPrefix, this.tableIdentifier, this.subtaskId, chkId);
if (!this.enable2PC) {
label = label + "_" + UUID.randomUUID();
}
if (LABEL_PATTERN.matcher(label).matches()) {
return label; // 前缀_table标识_subtaskID_ckID_UUID
} else {
return this.enable2PC ? String.format("%s_%s_%s_%s", this.labelPrefix, UUID.randomUUID(), this.subtaskId, chkId) : String.format("%s_%s_%s_%s", this.labelPrefix, this.subtaskId, chkId, UUID.randomUUID());
// 开启2pc:前缀_UUID_subtaskID_ckID
// 未开启2pc前缀_subtaskID_ckID_UUID
}
}
在 DorisStreamLoad
中,先判断是否需要写入行分隔符,再调用 RecordStream.write
将数据写入缓冲区:
kotlin
private final RecordStream recordStream;
public void writeRecord(byte[] record) throws InterruptedException {
this.checkLoadException(); // 检查异常情况
try {
if (this.loadBatchFirstRecord) {
this.loadBatchFirstRecord = false;
} else if (this.lineDelimiter != null) {
this.recordStream.write(this.lineDelimiter); // 一行一行写取
}
this.recordStream.write(record); // 调用RecordStream去写入
} catch (InterruptedException var3) {
Thread.currentThread().interrupt();
if (this.httpException != null) {
throw new DorisRuntimeException(this.httpException.getMessage(), this.httpException);
} else {
LOG.info("write record interrupted, cause " + var3.getClass());
throw var3;
}
}
}
<2> 缓冲区流转--RecordStream、RecordBuffer
RecordStream--InputStream的封装
java
public class RecordStream extends InputStream {
private final RecordBuffer recordBuffer;
public int read() throws IOException {
return 0;
}
public RecordStream(int bufferSize, int bufferCount, boolean useCache) {
if (useCache) {
this.recordBuffer = new CacheRecordBuffer(bufferSize, bufferCount);
} else {
this.recordBuffer = new RecordBuffer(bufferSize, bufferCount);
}
}
public void startInput(boolean isResume) throws IOException {
if (!isResume && this.recordBuffer instanceof CacheRecordBuffer) {
((CacheRecordBuffer)this.recordBuffer).recycleCache();
}
this.recordBuffer.startBufferData();
}
public void endInput() throws InterruptedException {
this.recordBuffer.stopBufferData();
}
public int read(byte[] buff) throws IOException {
try {
return this.recordBuffer.read(buff);
} catch (InterruptedException var3) {
Thread.currentThread().interrupt();
throw new RuntimeException(var3);
}
}
public void write(byte[] buff) throws InterruptedException {
try {
this.recordBuffer.write(buff); // 调用RecordBuffer去写入
} catch (InterruptedException var3) {
throw var3;
}
}
}
RecordBuffer--双队列+环形缓冲区
RecordBuffer
是核心缓冲区,用 writeQueue
(空闲缓冲区) 和 readQueue
(待发送缓冲区) 实现生产者 - 消费者模型:
kotlin
public class RecordBuffer {
private static final Logger LOG = LoggerFactory.getLogger(RecordBuffer.class);
BlockingQueue<ByteBuffer> writeQueue; // 写队列(里面每个元素都是一个缓冲区)
LinkedBlockingDeque<ByteBuffer> readQueue; // 读队列(里面每个元素都是一个缓冲区)
int bufferCapacity;
int queueSize;
ByteBuffer currentWriteBuffer;
ByteBuffer currentReadBuffer;
public RecordBuffer(int capacity, int queueSize) {
// 创建固定大小的写队列
this.writeQueue = new ArrayBlockingQueue(queueSize);
for(int index = 0; index < queueSize; ++index) {
this.writeQueue.add(ByteBuffer.allocate(capacity));// writeQueue是个队列,队列里面每个元素放的是缓冲区,预分配缓冲区
}
this.readQueue = new LinkedBlockingDeque();
this.bufferCapacity = capacity;
this.queueSize = queueSize;
}
public void startBufferData() throws IOException {
LOG.info("start buffer data, read queue size {}, write queue size {}", this.readQueue.size(), this.writeQueue.size());
Preconditions.checkState(this.readQueue.size() == 0);
Preconditions.checkState(this.writeQueue.size() == this.queueSize);
Iterator var1 = this.writeQueue.iterator();
while(var1.hasNext()) {
ByteBuffer byteBuffer = (ByteBuffer)var1.next();
Preconditions.checkState(byteBuffer.position() == 0);
Preconditions.checkState(byteBuffer.remaining() == this.bufferCapacity);
}
}
public void stopBufferData() throws InterruptedException {
boolean isEmpty = false;
if (this.currentWriteBuffer != null) {
this.currentWriteBuffer.flip(); // 将写缓冲区状态转换为读
isEmpty = this.currentWriteBuffer.limit() == 0;
this.readQueue.put(this.currentWriteBuffer);// 放到读队列中
this.currentWriteBuffer = null;
}
if (!isEmpty) {
ByteBuffer byteBuffer = (ByteBuffer)this.writeQueue.take();
byteBuffer.flip();
Preconditions.checkState(byteBuffer.limit() == 0);
this.readQueue.put(byteBuffer);
}
}
// 生产数据
public void write(byte[] buf) throws InterruptedException {
int wPos = 0;
do {
if (this.currentWriteBuffer == null) {
this.currentWriteBuffer = (ByteBuffer)this.writeQueue.take(); // 从写队列中获取缓冲区
}
int available = this.currentWriteBuffer.remaining();
int nWrite = Math.min(available, buf.length - wPos);
this.currentWriteBuffer.put(buf, wPos, nWrite); // 写入数据
wPos += nWrite;
if (this.currentWriteBuffer.remaining() == 0) { // 当前缓冲区满了
this.currentWriteBuffer.flip(); // 关键:切换为读模式
this.readQueue.put(this.currentWriteBuffer); // 放入到读队列,等待消费
this.currentWriteBuffer = null;
}
} while(wPos != buf.length);
}
public int read(byte[] buf) throws InterruptedException {
if (this.currentReadBuffer == null) {
this.currentReadBuffer = (ByteBuffer)this.readQueue.take(); // 从读队列获取缓冲区
}
if (this.currentReadBuffer.limit() == 0) { // 当前缓冲区已经读完了
this.recycleBuffer(this.currentReadBuffer); // 放回写队列
this.currentReadBuffer = null; // 滞空
Preconditions.checkState(this.readQueue.size() == 0);
return -1; // -1表示读取失败,无数据可读
} else {
int available = this.currentReadBuffer.remaining();
int nRead = Math.min(available, buf.length); // 计算最小可读数据的量
this.currentReadBuffer.get(buf, 0, nRead); // 读取数据
if (this.currentReadBuffer.remaining() == 0) { // 缓冲区读完了
this.recycleBuffer(this.currentReadBuffer); // 放回写队列
this.currentReadBuffer = null;
}
return nRead; // 返回读取的数据的量
}
}
// 将当前缓冲区重置,并放回到写队列
private void recycleBuffer(ByteBuffer buffer) throws InterruptedException {
buffer.clear();
this.writeQueue.put(buffer);
}
public int getWriteQueueSize() {
return this.writeQueue.size();
}
public int getReadQueueSize() {
return this.readQueue.size();
}
}
<3> 发送HTTP请求--InputStreamEntity + 线程池
(1)启动发送:DorisStreamLoad.startLoad
当触发 startLoad
(如缓冲区达到阈值、Flink 触发 checkpoint 等),会构建 HTTP 请求,并将 RecordStream
作为请求体:
kotlin
public void startLoad(String label, boolean isResume) throws IOException {
// 初始化RecordStream
this.loadBatchFirstRecord = !isResume;
HttpPutBuilder putBuilder = new HttpPutBuilder();
this.recordStream.startInput(isResume);
this.currentLabel = label;
String executeMessage;
try {
// 将RecordStream包装成http的请求体
InputStreamEntity entity = new InputStreamEntity(this.recordStream);
putBuilder.setUrl(this.loadUrlStr) // 设置url:http://%s/api/%s/%s/_stream_load,这个url在doris端被标记为未提交状态--prepare
.baseAuth(this.user,this.passwd)
.addCommonHeader()
.addHiddenColumns(this.enableDelete)
.setLabel(label) // 设置标签--- 这里的标签是由前缀_table标识_subtaskID_ckid组成,若开启事务还会添加uuid
.setEntity(entity)
.addProperties(this.streamLoadProp);
if (this.enable2PC) {
putBuilder.enable2PC(); // 底层就是携带http请求体参数:two_phase_commit=true
}
if (this.enableGzCompress) {
putBuilder.setEntity(new GzipCompressingEntity(entity));
}
if (this.enableGroupCommit) {
executeMessage = "table " + this.table + " start execute load with group commit";
} else {
executeMessage = "table " + this.table + " start execute load for label " + label;
}
// 开启线程,实现异步提交任务,发送HTTP请求
Thread mainThread = Thread.currentThread();
this.pendingLoadFuture = this.executorService.submit(() -> {
LOG.info(executeMessage);
// 执行(提交)HTTP请求
CloseableHttpResponse execute = this.httpClient.execute(putBuilder.build());
RespContent respContent = this.handlePreCommitResponse(execute); // 获取响应,开启事务会返回txnID
return respContent;
}
}
(2)请求体发送:InputStreamEntity.writeTo
HTTP开始execute的时候,底层会自动调用InputStreamEntity的writeTo(outputStream)
方法
InputStreamEntity
实现了 writeTo
方法,会循环调用 RecordStream.read
,从缓冲区读取数据并写入 HTTP 输出流:
java
public class InputStreamEntity extends AbstractHttpEntity {
private final InputStream content;
private final long length;
public InputStreamEntity(InputStream inStream) {
this(inStream, -1L);
}
public InputStreamEntity(InputStream inStream, long length) {
this(inStream, length, (ContentType)null);
}
public InputStreamEntity(InputStream inStream, ContentType contentType) {
this(inStream, -1L, contentType);
}
public InputStreamEntity(InputStream inStream, long length, ContentType contentType) {
this.content = (InputStream)Args.notNull(inStream, "Source input stream");
this.length = length;
if (contentType != null) {
this.setContentType(contentType.toString());
}
}
public boolean isRepeatable() {
return false;
}
public long getContentLength() {
return this.length;
}
public InputStream getContent() throws IOException {
return this.content;
}
public void writeTo(OutputStream outStream) throws IOException {
Args.notNull(outStream, "Output stream");
InputStream inStream = this.content;
try {
byte[] buffer = new byte[4096];
int readLen;
if (this.length < 0L) { // 长度未知,流式读取
while((readLen = inStream.read(buffer)) != -1) { // 直到读到末尾
outStream.write(buffer, 0, readLen); // 写入HTTP输出流
}
} else { // 长度已知,按长度读取,默认是4096
for(long remaining = this.length; remaining > 0L; remaining -= (long)readLen) {
readLen = inStream.read(buffer, 0, (int)Math.min(4096L, remaining));
if (readLen == -1) {
break;
}
outStream.write(buffer, 0, readLen);
}
}
} finally {
inStream.close();
}
}
public boolean isStreaming() {
return true;
}
}
(3)响应处理:DorisStreamLoad.handlePreCommitResponse
java
// DorisStreamLoad.java
public RespContent handlePreCommitResponse(CloseableHttpResponse response) throws Exception {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200 && response.getEntity() != null) { // 响应成功
String loadResult = EntityUtils.toString(response.getEntity());
RespContent respContent = OBJECT_MAPPER.readValue(loadResult, RespContent.class);
if (respContent.getStatus().equals("Label Already Exists")) { // 处理 Label 冲突
abortLabelExistTransaction(respContent); // 回滚事务
throw new LabelAlreadyExistsException("重试...");
}
return respContent;
}
// 响应失败,直接throw异常
throw new StreamLoadException("请求失败...");
}
(2) 预提交源码分析
<1> prepareCommit
- 结束当前批次的数据写入 :将
RecordBuffer
中剩余的数据发送到 Doris。 - 生成提交信息 :为每个表创建
DorisCommittable
,包含事务 ID 等信息,供后续Committer
提交事务。 - 重置状态:清理当前批次的加载状态,为下一批次做准备。
scss
public Collection<DorisCommittable> prepareCommit() throws IOException, InterruptedException {
// 1. 检查是否有数据需要提交
if (!globalLoading && loadingMap.values().stream().noneMatch(Boolean::booleanValue)) {
return Collections.emptyList();
}
// 2. 标记全局加载状态为 false
this.globalLoading = false;
List<DorisCommittable> committableList = new ArrayList();
// 3. 遍历所有表的 DorisStreamLoad
for (Map.Entry<String, DorisStreamLoad> streamLoader : dorisStreamLoadMap.entrySet()) {
String tableIdentifier = streamLoader.getKey();
if (!loadingMap.getOrDefault(tableIdentifier, false)) {
continue; // 跳过无数据的表
}
// 4. 停止当前数据的加载,获取响应
DorisStreamLoad dorisStreamLoad = streamLoader.getValue();
RespContent respContent = dorisStreamLoad.stopLoad();
// 5. 更新指标
if (sinkMetricsMap.containsKey(tableIdentifier)) {
sinkMetricsMap.get(tableIdentifier).flush(respContent);
}
// 6. 如果启用了 2PC,创建 Committable 记录事务 ID,在后续DorisCommiter中会根据txnid、host port、db去发送http请求commit
if (executionOptions.enabled2PC()) {
long txnId = respContent.getTxnId();
committableList.add(new DorisCommittable(
dorisStreamLoad.getHostPort(),
dorisStreamLoad.getDb(),
txnId
));
}
}
// 7. 清理当前批次的加载状态
loadingMap.clear();
return committableList;
}
<2> DorisStreamLoad.stopLoad
stopLoad()
是 DorisStreamLoad
类中的关键方法,负责结束当前批次的数据写入 并等待 Doris 响应。结合源码,我来解释其核心逻辑:
- 关闭数据输入 :通知
RecordStream
停止接收新数据。 - 等待 HTTP 请求完成:阻塞等待异步任务返回 Doris 的响应结果。
- 处理异常:若发送过程中出现异常,捕获并抛出。
kotlin
public RespContent stopLoad() throws InterruptedException {
try {
//1.结束 RecordStream 的输入,标记数据已写完
this.recordStream.endInput();
// 2. 记录日志
if (this.enableGroupCommit) {
LOG.info("table {} stream load stopped with group commit on host {}", this.table, this.hostPort);
} else {
LOG.info("table {} stream load stopped for {} on host {}", new Object[]{this.table, this.currentLabel, this.hostPort});
}
Preconditions.checkState(this.pendingLoadFuture != null);
// 3. 阻塞等待异步任务HTTP请求完成,获取 Doris 响应
return (RespContent)this.pendingLoadFuture.get();
} catch (InterruptedException var2) {
Thread.currentThread().interrupt();
if (this.httpException != null) {
throw new DorisRuntimeException(this.httpException.getMessage(), this.httpException);
} else {
throw var2;
}
} catch (ExecutionException var3) {
throw new DorisRuntimeException(var3);
}
}
<3> endInput停止写入数据
kotlin
// RecordStream的endInput
public void endInput() throws InterruptedException {
this.recordBuffer.stopBufferData();
}
// 调用RecordBuffer的stopBufferData
public void stopBufferData() throws InterruptedException {
boolean isEmpty = false;
if (this.currentWriteBuffer != null) {
this.currentWriteBuffer.flip();// 切换为读模式
isEmpty = this.currentWriteBuffer.limit() == 0;
this.readQueue.put(this.currentWriteBuffer);
this.currentWriteBuffer = null;
}
if (!isEmpty) {
ByteBuffer byteBuffer = (ByteBuffer)this.writeQueue.take();
byteBuffer.flip();
Preconditions.checkState(byteBuffer.limit() == 0);
this.readQueue.put(byteBuffer);
}
}
(3) snapshot源码分析
<1> 整体逻辑
DorisWriter.snapshotState()
是 Flink 检查点机制的核心方法,负责保存当前写入器的状态,确保在故障恢复后能从断点继续处理。结合源码,我来解释其核心逻辑:
- 保存关键状态:记录每个表的写入进度、事务信息等。
- 更新可用后端:为每个表选择新的可用 Doris 节点。
- 推进检查点 ID:更新下一个检查点的 ID,确保事务标签的唯一性。
java
public List<DorisWriterState> snapshotState(long checkpointId) throws IOException {
List<DorisWriterState> writerStates = new ArrayList();
// 1. 遍历所有表的 DorisStreamLoad
for (DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()) {
// 2. 为每个表选择新的可用后端,从DorisFE节点中获取监控的BE节点
dorisStreamLoad.setHostPort(backendUtil.getAvailableBackend(subtaskId));
// 3. 创建并保存状态对象
DorisWriterState writerState = new DorisWriterState(
labelPrefix, // 标签前缀由
dorisStreamLoad.getDb(), // 数据库名
dorisStreamLoad.getTable(), // 表名
subtaskId // 子任务 ID
);
writerStates.add(writerState);
}
// 4. 更新下一个检查点 ID
this.curCheckpointId = checkpointId + 1L;
return writerStates;
}
<2> 获取健康的BE节点--backendUtil.getAvailableBackend(subtaskId)
他会根据subtask_id去获取对应健康的BE节点
arduino
public String getAvailableBackend(int subtaskId) {
// 计算最大尝试次数(当前节点列表长度)
long tmp = this.pos + (long)this.backends.size();
String res;
// 循环尝试,直到找到可用节点或达到最大尝试次数
do {
// 若尝试次数超过节点数,抛出异常
if (this.pos >= tmp) {
throw new DorisRuntimeException("no available backend.");
}
// 计算当前尝试的节点索引(subtaskId 参与哈希,确保不同子任务选择不同节点)
BackendV2.BackendRowV2 backend = (BackendV2.BackendRowV2)this.backends.get(
(int)((this.pos + (long)subtaskId) % (long)this.backends.size())
);
++this.pos; // 尝试位置递增
// 获取节点地址(格式:host:port)
res = backend.toBackendString();
// 检查节点是否可连接(核心健康检查逻辑)
} while(!tryHttpConnection(res));
return res; // 返回可用节点地址
}
<3> 给DorisStreamLoad配置host port
这个方法在后续的checkAllDone还会被调用,但目前checkDone已弃用了,改用CompletableFuture和submit异步任务监听了
checkDone设计缺陷
轮询机制低效 :checkDone
采用定时轮询(intervalTime
)检查任务状态,属于被动式监控,消耗 CPU 资源且响应不及时。
耦合度高 :与 globalLoading
和 loadingMap
强耦合,增加了代码维护复杂度。
kotlin
public void setHostPort(String hostPort) {
this.hostPort = hostPort;
// 更新数据加载 URL(用于发送数据)
this.loadUrlStr = String.format("http://%s/api/%s/%s/_stream_load", hostPort, this.db, this.table);
// 更新事务控制 URL(用于 2PC 模式下的提交/回滚)
this.abortUrlStr = String.format("http://%s/api/%s/_stream_load_2pc", hostPort, this.db);
}
4.DorisCommitter
kotlin
public class DorisCommitter implements Committer<DorisCommittable>, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(DorisCommitter.class);
private static final String commitPattern = "http://%s/api/%s/_stream_load_2pc";
private final CloseableHttpClient httpClient;
private final DorisOptions dorisOptions;
private final DorisReadOptions dorisReadOptions;
private final ObjectMapper jsonMapper;
private final BackendUtil backendUtil;
int maxRetry;
final boolean ignoreCommitError;
public DorisCommitter(DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, DorisExecutionOptions executionOptions) {
this(dorisOptions, dorisReadOptions, executionOptions, (new HttpUtil(dorisReadOptions)).getHttpClient());
}
public DorisCommitter(DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, DorisExecutionOptions executionOptions, CloseableHttpClient client) {
this.jsonMapper = new ObjectMapper();
this.dorisOptions = dorisOptions;
this.dorisReadOptions = dorisReadOptions;
Preconditions.checkArgument(this.maxRetry >= 0);
this.maxRetry = executionOptions.getMaxRetries();
this.ignoreCommitError = executionOptions.ignoreCommitError();
this.httpClient = client;
this.backendUtil = StringUtils.isNotEmpty(dorisOptions.getBenodes()) ? new BackendUtil(dorisOptions.getBenodes()) : new BackendUtil(RestService.getBackendsV2(dorisOptions, dorisReadOptions, LOG));
}
public void commit(Collection<Committer.CommitRequest<DorisCommittable>> requests) throws IOException, InterruptedException {
// 这里的requests是DorisWriter中的prepareCommit方法创建的
Iterator var2 = requests.iterator();
while(var2.hasNext()) {
Committer.CommitRequest<DorisCommittable> request = (Committer.CommitRequest)var2.next();
this.commitTransaction((DorisCommittable)request.getCommittable());
}
}
private void commitTransaction(DorisCommittable committable) throws IOException {
// 创造http请求,往http://%s/api/%s/_stream_load_2pc发送commit
HttpPutBuilder builder = (new HttpPutBuilder())
.addCommonHeader()
.baseAuth(this.dorisOptions.getUsername(), this.dorisOptions.getPassword())
.addTxnId(committable.getTxnID()) // 携带事务id--txnID
.commit();
String hostPort = committable.getHostPort();
LOG.info("commit txn {} to host {}", committable.getTxnID(), hostPort);
Throwable ex = new Throwable();
int retry = 0;
while(retry <= this.maxRetry) {
String url = String.format("http://%s/api/%s/_stream_load_2pc", hostPort, committable.getDb());
HttpPut httpPut = builder
.setUrl(url)
.setEmptyEntity()
.build();
try {
label170: {
// 1.执行HTTP请求提交事务,返回响应
CloseableHttpResponse response = this.httpClient.execute(httpPut);
Throwable var9 = null;
try {
// 2.解析响应状态行(状态码+状态信息)
StatusLine statusLine = response.getStatusLine();
String loadResult;
// 3.判断状态码非200 || 响应体为空的情况
if (200 != statusLine.getStatusCode() || response.getEntity() == null) {
loadResult = statusLine.getReasonPhrase();
LOG.error("commit failed with {}, reason {}", hostPort, loadResult);
// 3.1 重试次数已经用完,封装异常
if (retry == this.maxRetry) {
ex = new DorisRuntimeException("commit transaction error: " + loadResult);
}
// 3.2 还可以重试,重新获取一个可用的BE节点,再次commit
hostPort = this.backendUtil.getAvailableBackend();
break label170;
}
// 4. 状态码 200 且有响应体,解析响应内容
loadResult = EntityUtils.toString(response.getEntity());
//4.1 把 Doris 返回的 JSON 字符串转成 Map(比如 { "status": "Success", "msg": "..." } )
Map<String, String> res = (Map)this.jsonMapper.readValue(
loadResult, new TypeReference<HashMap<String, String>>() {}
);
// 5. 判断 Doris 返回的状态是否成功
if (((String)res.get("status")).equals("Success")) {
LOG.info("load result {}", loadResult);
} else {
// 5.1 检查响应是否已经是"提交成功"状态(防止重复提交)
if (!ResponseUtil.isCommitted((String)res.get("msg"))) {
throw new DorisRuntimeException("commit transaction failed " + loadResult);
}
// 5.2 已经提交过了,打日志跳过
LOG.info("transaction {} has already committed successfully, skipping, load response is {}", committable.getTxnID(), res.get("msg"));
}
} catch (Throwable var23) {
// 6. 捕获 try 块里的异常,给 var9 标记,继续往上抛
var9 = var23;
throw var23;
} finally {
// 7. 无论成功/失败,都要关闭响应(释放连接)
if (response != null) {
if (var9 != null) {
try {
response.close();
} catch (Throwable var22) {
var9.addSuppressed(var22);
}
} else {
response.close();
}
}
}
return;
}
} catch (Exception var25) {
LOG.error("commit transaction failed, to retry, {}", var25.getMessage());
ex = var25;
hostPort = this.backendUtil.getAvailableBackend();
}
if (retry++ >= this.maxRetry) {
if (!this.ignoreCommitError) {
throw new DorisRuntimeException("commit transaction error, ", (Throwable)ex);
}
LOG.error("Unable to commit transaction {} and data has been potentially lost ", committable, ex);
}
}
}
public void close() {
if (this.httpClient != null) {
try {
this.httpClient.close();
} catch (IOException var2) {
}
}
}
}
5.总结
- 数据写入的时候是写入到环形缓冲区的写队列缓冲区中,当缓冲区满了,会把它转为读,放到读队列中,然后开启异步线程去execute发送HTTP请求到http://hostPort/api/db/table/streamload,数据就是这个读队列的缓冲区的所有数据
- 数据做prepareCommit的时候,停止数据写入了,并把剩余数据转为读状态,加入到读队列中,并阻塞线程等待之前异步http请求的线程返回响应,然后根据响应去执行相应的操作
- 数据在做snapshot的时候,只需要遍历所有表的写入器DorisStreamLoad,获取健康的BE节点,然后给DorisStreamLoad写入器配置对应的host port,然后将labelprefix、db、table、subtaskid存到状态中,
- 做完ck后,数据开始正式提交,通过DorisCommitter去遍历committableList去根据txnId去提交commit到http://hostPort/api/db/streamload_2pc