nacos配置监听设计
一. nacos配置监听策略
1.策略
- V1版本的是采用http长轮询策略
- V2版本的是grpc轮询
2.原理
- V1版本http长轮询，客户端定时请求配置中心，设置较长的http请求超时时间，服务端收到请求之后，主线程挂起暂时不返回，将请求放到线程池执行；其中核心是利用了servlet3.0的AsyncContext实现延时返回，降低配置中心的压力。
- V2版本，利用grpc通讯，实现主机之间的全双工通讯。
3.部分实现代码逻辑
（1）V1版本长轮询
监听配置修改
- 客户端请求服务端的接口
bash
/**
* V1版本的nacos-client请求到该接口
*/
@PostMapping("/listener")
@Secured(action = ActionTypes.READ, signType = SignType.CONFIG)
@ExtractorManager.Extractor(httpExtractor = ConfigListenerHttpParamExtractor.class)
public void listener(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
request.setAttribute("org.apache.catalina.ASYNC_SUPPORTED", true);
String probeModify = request.getParameter("Listening-Configs");
if (StringUtils.isBlank(probeModify)) {
LOGGER.warn("invalid probeModify is blank");
throw new IllegalArgumentException("invalid probeModify");
}
probeModify = URLDecoder.decode(probeModify, Constants.ENCODE);
Map<String, String> clientMd5Map;
try {
clientMd5Map = MD5Util.getClientMd5Map(probeModify);
} catch (Throwable e) {
throw new IllegalArgumentException("invalid probeModify");
}
// do long-polling
inner.doPollingConfig(request, response, clientMd5Map, probeModify.length());
}- 添加到线程池
bash
/**
*
*/
public String doPollingConfig(HttpServletRequest request, HttpServletResponse response,
Map<String, String> clientMd5Map, int probeRequestSize) throws IOException {
// Long polling.
if (LongPollingService.isSupportLongPolling(request)) {//如果请求header的Long-Pulling-Timeout（客户端http请求超时时间）不为空，则添加到线程池
longPollingService.addLongPollingClient(request, response, clientMd5Map, probeRequestSize);
return HttpServletResponse.SC_OK + "";
}
// Compatible with short polling logic.
List<String> changedGroups = MD5Util.compareMd5(request, response, clientMd5Map);
// Compatible with short polling result.
String oldResult = MD5Util.compareMd5OldResult(changedGroups);
String newResult = MD5Util.compareMd5ResultString(changedGroups);
String version = request.getHeader(Constants.CLIENT_VERSION_HEADER);
if (version == null) {
version = "2.0.0";
}
int versionNum = Protocol.getVersionNumber(version);
// Before 2.0.4 version, return value is put into header.
if (versionNum < START_LONG_POLLING_VERSION_NUM) {
response.addHeader(Constants.PROBE_MODIFY_RESPONSE, oldResult);
response.addHeader(Constants.PROBE_MODIFY_RESPONSE_NEW, newResult);
} else {
request.setAttribute("content", newResult);
}
// Disable cache.
response.setHeader("Pragma", "no-cache");
response.setDateHeader("Expires", 0);
response.setHeader("Cache-Control", "no-cache,no-store");
response.setStatus(HttpServletResponse.SC_OK);
return HttpServletResponse.SC_OK + "";
}
bash
/**
* Add LongPollingClient.
*
* @param req HttpServletRequest.
* @param rsp HttpServletResponse.
* @param clientMd5Map clientMd5Map.
* @param probeRequestSize probeRequestSize.
*/
public void addLongPollingClient(HttpServletRequest req, HttpServletResponse rsp, Map<String, String> clientMd5Map,
int probeRequestSize) {
String noHangUpFlag = req.getHeader(LongPollingService.LONG_POLLING_NO_HANG_UP_HEADER);
long start = System.currentTimeMillis();
List<String> changedGroups = MD5Util.compareMd5(req, rsp, clientMd5Map);
if (changedGroups.size() > 0) {//如果存在文件修改了，则立即返回
generateResponse(req, rsp, changedGroups);//返回修改
LogUtil.CLIENT_LOG.info("{}|{}|{}|{}|{}|{}|{}", System.currentTimeMillis() - start, "instant",
RequestUtil.getRemoteIp(req), "polling", clientMd5Map.size(), probeRequestSize,
changedGroups.size());
return;
} else if (noHangUpFlag != null && noHangUpFlag.equalsIgnoreCase(TRUE_STR)) {
LogUtil.CLIENT_LOG.info("{}|{}|{}|{}|{}|{}|{}", System.currentTimeMillis() - start, "nohangup",
RequestUtil.getRemoteIp(req), "polling", clientMd5Map.size(), probeRequestSize,
changedGroups.size());
return;
}
// Must be called by http thread, or send response.
final AsyncContext asyncContext = req.startAsync();//servlet3.0的AsyncContext，可以异步执行
// AsyncContext.setTimeout() is incorrect, Control by oneself
asyncContext.setTimeout(0L);
String ip = RequestUtil.getRemoteIp(req);
ConnectionCheckResponse connectionCheckResponse = checkLimit(req);//检查链连
if (!connectionCheckResponse.isSuccess()) {
RpcScheduledExecutor.CONTROL_SCHEDULER.schedule(new Runnable() {
@Override
public void run() {
generate503Response(asyncContext, rsp, connectionCheckResponse.getMessage());
}
}, 1000L + new Random().nextInt(2000), TimeUnit.MILLISECONDS);
return;
}
String appName = req.getHeader(RequestUtil.CLIENT_APPNAME_HEADER);
String tag = req.getHeader("Vipserver-Tag");
int delayTime = SwitchService.getSwitchInteger(SwitchService.FIXED_DELAY_TIME, 500);//随机生成提前返回结果的时长，500ms内
int minLongPoolingTimeout = SwitchService.getSwitchInteger("MIN_LONG_POOLING_TIMEOUT", 10000);//最小延时返回时长
// Add delay time for LoadBalance, and one response is returned 500 ms in advance to avoid client timeout.
String requestLongPollingTimeOut = req.getHeader(LongPollingService.LONG_POLLING_HEADER);//获取客户端的请求超时时长
long timeout = Math.max(minLongPoolingTimeout, Long.parseLong(requestLongPollingTimeOut) - delayTime);//服务端延时返回时长
ConfigExecutor.executeLongPolling(
new ClientLongPolling(asyncContext, clientMd5Map, ip, probeRequestSize, timeout, appName, tag));//放到线程池执行
}- ClientLongPolling 处理
java
public class ClientLongPolling implements Runnable {
final AsyncContext asyncContext;
final Map<String, String> clientMd5Map;
final long createTime;
final String ip;
final String appName;
final String tag;
final int probeRequestSize;
final long timeoutTime;//服务端延时返回时长
Future<?> asyncTimeoutFuture;
@Override
public void run() {
//创建一个延时任务，延时删除ClientLongPolling这个任务
asyncTimeoutFuture = ConfigExecutor.scheduleLongPolling(() -> {
try {
getRetainIps().put(ClientLongPolling.this.ip, System.currentTimeMillis());
// Delete subscriber's relations. //allSubs
boolean removeFlag = allSubs.remove(ClientLongPolling.this);
if (removeFlag) {//如果到了执行时间，并且任务还在队列中，则正常就是配置没有修改，则调用sendResponse(null);
LogUtil.CLIENT_LOG.info("{}|{}|{}|{}|{}|{}", (System.currentTimeMillis() - createTime),
"timeout", RequestUtil.getRemoteIp((HttpServletRequest) asyncContext.getRequest()),
"polling", clientMd5Map.size(), probeRequestSize);
sendResponse(null);//直接返回http请求结果，响应结果为空
} else {
LogUtil.DEFAULT_LOG.warn("client subsciber's relations delete fail.");
}
} catch (Throwable t) {
LogUtil.DEFAULT_LOG.error("long polling error:" + t.getMessage(), t.getCause());
}
}, timeoutTime, TimeUnit.MILLISECONDS);
allSubs.add(this);
}
void sendResponse(List<String> changedGroups) {
// Cancel time out task.
if (null != asyncTimeoutFuture) {
asyncTimeoutFuture.cancel(false);
}
generateResponse(changedGroups);
}
void generateResponse(List<String> changedGroups) {
if (null == changedGroups) {
// Tell web container to send http response.
asyncContext.complete();
return;
}
HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse();
try {
final String respString = MD5Util.compareMd5ResultString(changedGroups);
// Disable cache.
response.setHeader("Pragma", "no-cache");
response.setDateHeader("Expires", 0);
response.setHeader("Cache-Control", "no-cache,no-store");
response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().println(respString);
asyncContext.complete();//结束http异步请求的
} catch (Exception ex) {
PULL_LOG.error(ex.toString(), ex);
asyncContext.complete();
}
}
ClientLongPolling(AsyncContext ac, Map<String, String> clientMd5Map, String ip, int probeRequestSize,
long timeoutTime, String appName, String tag) {
this.asyncContext = ac;
this.clientMd5Map = clientMd5Map;
this.probeRequestSize = probeRequestSize;
this.createTime = System.currentTimeMillis();
this.ip = ip;
this.timeoutTime = timeoutTime;
this.appName = appName;
this.tag = tag;
}
@Override
public String toString() {
return "ClientLongPolling{" + "clientMd5Map=" + clientMd5Map + ", createTime=" + createTime + ", ip='" + ip
+ '\'' + ", appName='" + appName + '\'' + ", tag='" + tag + '\'' + ", probeRequestSize="
+ probeRequestSize + ", timeoutTime=" + timeoutTime + '}';
}
}- 数据修改任务
java
//客户端所有的订阅请求任务队列
final Queue<ClientLongPolling> allSubs;
class DataChangeTask implements Runnable {//
@Override
public void run() {
try {
for (Iterator<ClientLongPolling> iter = allSubs.iterator(); iter.hasNext(); ) {//遍历队列
ClientLongPolling clientSub = iter.next();
if (clientSub.clientMd5Map.containsKey(groupKey)) {
// If published tag is not in the tag list, then it skipped.
if (StringUtils.isNotBlank(tag) && !tag.equals(clientSub.tag)) {
continue;
}
getRetainIps().put(clientSub.ip, System.currentTimeMillis());
iter.remove(); // Delete subscribers' relationships.
LogUtil.CLIENT_LOG.info("{}|{}|{}|{}|{}|{}|{}", (System.currentTimeMillis() - changeTime),
"in-advance",
RequestUtil.getRemoteIp((HttpServletRequest) clientSub.asyncContext.getRequest()),
"polling", clientSub.clientMd5Map.size(), clientSub.probeRequestSize, groupKey);
clientSub.sendResponse(Collections.singletonList(groupKey));//返回客户端修改了的groupkey
}
}
} catch (Throwable t) {
LogUtil.DEFAULT_LOG.error("data change error: {}", ExceptionUtil.getStackTrace(t));
}
}
DataChangeTask(String groupKey, boolean isBeta, List<String> betaIps) {
this(groupKey, isBeta, betaIps, null);
}
DataChangeTask(String groupKey, boolean isBeta, List<String> betaIps, String tag) {
this.groupKey = groupKey;
this.isBeta = isBeta;
this.betaIps = betaIps;
this.tag = tag;
}
final String groupKey;
final long changeTime = System.currentTimeMillis();
final boolean isBeta;
final List<String> betaIps;
final String tag;
}- DataChangeTask 在收到LocalDataChangeEvent 事件更改时，就会触发，这个时候调用到ClientLongPolling中的sendResponse方法，及时响应到客户端配置文件的修改
java
@SuppressWarnings("PMD.ThreadPoolCreationRule")
public LongPollingService() {//构造函数，启动时就初始化 "allSubs" 队列信息更新校验任务、注册配置更新通知、订阅事件
allSubs = new ConcurrentLinkedQueue<>();
ConfigExecutor.scheduleLongPolling(new StatTask(), 0L, 10L, TimeUnit.SECONDS);
// Register LocalDataChangeEvent to NotifyCenter.
NotifyCenter.registerToPublisher(LocalDataChangeEvent.class, NotifyCenter.ringBufferSize);
// Register A Subscriber to subscribe LocalDataChangeEvent.
NotifyCenter.registerSubscriber(new Subscriber() {
@Override
public void onEvent(Event event) {
if (event instanceof LocalDataChangeEvent) {
LocalDataChangeEvent evt = (LocalDataChangeEvent) event;
//配置更新之后，会丢到线程池调用sendResponse结束 AsyncContext 的请求
ConfigExecutor.executeLongPolling(new DataChangeTask(evt.groupKey, evt.isBeta, evt.betaIps));
}
}
@Override
public Class<? extends Event> subscribeType() {
return LocalDataChangeEvent.class;
}
});
}配置修改并发布事件
- 服务端修改配置之后调用接口 /nacos/v1/cs/configs
java
/**
* Adds or updates non-aggregated data.
* <p>
* request and response will be used in aspect, see
* {@link com.alibaba.nacos.config.server.aspect.CapacityManagementAspect} and
* {@link com.alibaba.nacos.config.server.aspect.RequestLogAspect}.
* </p>
*
* @throws NacosException NacosException.
*/
@PostMapping
@TpsControl(pointName = "ConfigPublish")
@Secured(action = ActionTypes.WRITE, signType = SignType.CONFIG)
public Boolean publishConfig(HttpServletRequest request, HttpServletResponse response,
@RequestParam(value = "dataId") String dataId, @RequestParam(value = "group") String group,
@RequestParam(value = "tenant", required = false, defaultValue = StringUtils.EMPTY) String tenant,
@RequestParam(value = "content") String content, @RequestParam(value = "tag", required = false) String tag,
@RequestParam(value = "appName", required = false) String appName,
@RequestParam(value = "src_user", required = false) String srcUser,
@RequestParam(value = "config_tags", required = false) String configTags,
@RequestParam(value = "desc", required = false) String desc,
@RequestParam(value = "use", required = false) String use,
@RequestParam(value = "effect", required = false) String effect,
@RequestParam(value = "type", required = false) String type,
@RequestParam(value = "schema", required = false) String schema,
@RequestParam(required = false) String encryptedDataKey) throws NacosException {
String encryptedDataKeyFinal = null;
if (StringUtils.isNotBlank(encryptedDataKey)) {
encryptedDataKeyFinal = encryptedDataKey;
} else {
Pair<String, String> pair = EncryptionHandler.encryptHandler(dataId, content);
content = pair.getSecond();
encryptedDataKeyFinal = pair.getFirst();
}
// check tenant
ParamUtils.checkTenant(tenant);
ParamUtils.checkParam(dataId, group, "datumId", content);
ParamUtils.checkParam(tag);
ConfigForm configForm = new ConfigForm();
configForm.setDataId(dataId);
configForm.setGroup(group);
configForm.setNamespaceId(tenant);
configForm.setContent(content);
configForm.setTag(tag);
configForm.setAppName(appName);
configForm.setSrcUser(srcUser);
configForm.setConfigTags(configTags);
configForm.setDesc(desc);
configForm.setUse(use);
configForm.setEffect(effect);
configForm.setType(type);
configForm.setSchema(schema);
if (StringUtils.isBlank(srcUser)) {
configForm.setSrcUser(RequestUtil.getSrcUserName(request));
}
if (!ConfigType.isValidType(type)) {
configForm.setType(ConfigType.getDefaultType().getType());
}
ConfigRequestInfo configRequestInfo = new ConfigRequestInfo();
configRequestInfo.setSrcIp(RequestUtil.getRemoteIp(request));
configRequestInfo.setRequestIpApp(RequestUtil.getAppName(request));
configRequestInfo.setBetaIps(request.getHeader("betaIps"));
configRequestInfo.setCasMd5(request.getHeader("casMd5"));
//发布配置修改
return configOperationService.publishConfig(configForm, configRequestInfo, encryptedDataKeyFinal);
}- 后续经过一系列校验等，调用发布配置修改的事件
java
ConfigChangePublisher.notifyConfigChange(
new ConfigDataChangeEvent(true, configForm.getDataId(), configForm.getGroup(),
configForm.getNamespaceId(), configOperateResult.getLastModified()));（2）V2版本grpc通讯
[TODO]
附录
参考资料
其他
nacos事件管理器 NotifyCenter（TODO）