6、NacosDistro协议

原创已于 2024-12-10 17:46:04 修改 · 583 阅读

5 ·

本内容遵循CC 4.0 BY-SA版权协议

GEO检测

收录于

Nacos

于 2024-12-06 15:21:32 首次发布

1、新节点同步数据

新Nacos节点加入集群，启动数据校验定时任务，默认间隔5秒，启动数据同步任务，拉取全量临时实例数据更新到本地缓存。
数据校验的心跳任务，将当前Nacos节点维护客户端最新版本号发送到Nacos集群其它节点，判断是否数据一致。
客户端与某个Nacos节点建立，isNative等于true，客户端数据同步到Nacos集群其它节点，isNative等于false。

    public DistroProtocol(ServerMemberManager memberManager, DistroComponentHolder distroComponentHolder,
            DistroTaskEngineHolder distroTaskEngineHolder) {
        this.memberManager = memberManager;
        this.distroComponentHolder = distroComponentHolder;
        this.distroTaskEngineHolder = distroTaskEngineHolder;
        startDistroTask();
    }
    private void startDistroTask() {
        if (EnvUtil.getStandaloneMode()) {
            isInitialized = true;
            return;
        }
        //数据校验
        startVerifyTask();
        //数据同步
        startLoadTask();
    }

    //执行数据校验任务
    private void startVerifyTask() {
		GlobalExecutor.schedulePartitionDataTimedSync(
	    	new DistroVerifyTimedTask(memberManager,distroComponentHolder,
       					distroTaskEngineHolder.getExecuteWorkersManager()),
         //任务间隔时间，默认间隔5秒
      	 DistroConfig.getInstance().getVerifyIntervalMillis());
    }
    
	//执行初始化任务加载
    private void startLoadTask() {
        DistroCallback loadCallback = new DistroCallback() {
            @Override
            public void onSuccess() {
                isInitialized = true;
            }
            @Override
            public void onFailed(Throwable throwable) {
                isInitialized = false;
            }
        };
        GlobalExecutor.submitLoadDataTask(new DistroLoadDataTask(memberManager, distroComponentHolder, 
                                         DistroConfig.getInstance(), loadCallback));
    }

数据校验任务startVerifyTask() 其它Nacos节点的客户端版本号与校验的版本号是否一致，不一致将最新数据推送到Nacos节点。
数据加载任务startLoadTask() 从其他节点同步数据到本节点，初始化状态isInitialized 数据同步成功后回调设置初始化为true。
全量拉取操作完成之后，每个Nacos节点上都持有当前的所有建立连接的客户端数据。

2、校验任务

DistroVerifyTimedTask 执行定时任务，定期发送心跳用于数据验证，心跳包含客户端的元信息，如果某个节点的数据与本地数据不一致，则会发送客户端全量的数据将数据补齐‌。

public void run() {
    try {
        //获取集群中除自已以外的节点
        List<Member> targetServer = serverMemberManager.allMembersWithoutSelf();
        if (Loggers.DISTRO.isDebugEnabled()) {
            Loggers.DISTRO.debug("server list is: {}", targetServer);
        }
        for (String each : distroComponentHolder.getDataStorageTypes()) {
            //执行数据校验请求
            verifyForDataStorage(each, targetServer);
        }
    } catch (Exception e) {
        Loggers.DISTRO.error("[DISTRO-FAILED] verify task failed.", e);
    }
}

private void verifyForDataStorage(String type, List<Member> targetServer) {
    //获取存储源数据
    DistroDataStorage dataStorage = distroComponentHolder.findDataStorage(type);
    if (!dataStorage.isFinishInitial()) {
        Loggers.DISTRO.warn("data storage {} has not finished initial step, do not send verify data",
                dataStorage.getClass().getSimpleName());
        return;
    }
    //获取当前节点维护的客户端数据，包含ClientId和版本号
    List<DistroData> verifyData = dataStorage.getVerifyData();
    if (null == verifyData || verifyData.isEmpty()) {
        return;
    }
    //通过任务向集群其它节点发送校验请求
    for (Member member : targetServer) {
        DistroTransportAgent agent = distroComponentHolder.findTransportAgent(type);
        if (null == agent) {
            continue;
        }
        executeTaskExecuteEngine.addTask(member.getAddress() + type,
                new DistroVerifyExecuteTask(agent, verifyData, member.getAddress(), type));
    }
}

3、同步任务

DistroLoadDataTask 执行数据的同步任务，遍历集群的节点发送同步数据请求拉取全量数据，响应成功则不发送给所有节点。

public class DistroLoadDataTask implements Runnable {
    //服务节点管理器
    private final ServerMemberManager memberManager;
    //协议持有者
    private final DistroComponentHolder distroComponentHolder;
    //协议配置
    private final DistroConfig distroConfig;
    //回调函数
    private final DistroCallback loadCallback;
    private final Map<String, Boolean> loadCompletedMap;
    
    public DistroLoadDataTask(ServerMemberManager memberManager, DistroComponentHolder 	
        distroComponentHolder, DistroConfig distroConfig, DistroCallback loadCallback) {
        this.memberManager = memberManager;
        this.distroComponentHolder = distroComponentHolder;
        this.distroConfig = distroConfig;
        this.loadCallback = loadCallback;
        loadCompletedMap = new HashMap<>(1);
    }
    
    @Override
    public void run() {
        try {
            //执行加载任务
            load();
           //判断是否加载完成，否则延迟30秒再执行加载
            if (!checkCompleted()) {
                //继续执行加载任务
                GlobalExecutor.submitLoadDataTask(this, distroConfig.getLoadDataRetryDelayMillis());
            } else {
                //执行加载成功的回调
                loadCallback.onSuccess();
            }
        } catch (Exception e) {
            //执行加载失败的回调
            loadCallback.onFailed(e);
        }
    }
    
    private void load() throws Exception {
        //集群节点为空则休眠1秒，其他节点可能还未启动
        while (memberManager.allMembersWithoutSelf().isEmpty()) {
            Loggers.DISTRO.info("[DISTRO-INIT] waiting server list init...");
            TimeUnit.SECONDS.sleep(1);
        }
        //组件注册器还未初始化
        while (distroComponentHolder.getDataStorageTypes().isEmpty()) {
            Loggers.DISTRO.info("[DISTRO-INIT] waiting distro data storage register...");
            TimeUnit.SECONDS.sleep(1);
        }
        //加载组件类型的数据
        for (String each : distroComponentHolder.getDataStorageTypes()) {
            if (!loadCompletedMap.containsKey(each) || !loadCompletedMap.get(each)) {
                //加载数据并进行标记
                loadCompletedMap.put(each, loadAllDataSnapshotFromRemote(each));
            }
        }
    }
    
    private boolean loadAllDataSnapshotFromRemote(String resourceType) {
        DistroTransportAgent transportAgent = distroComponentHolder.findTransportAgent(resourceType);
        DistroDataProcessor dataProcessor = distroComponentHolder.findDataProcessor(resourceType);
        if (null == transportAgent || null == dataProcessor) {
            return false;
        }
        for (Member each : memberManager.allMembersWithoutSelf()) {
            try {
                //从其它节点获取数据
                DistroData distroData = transportAgent.getDatumSnapshot(each.getAddress());
                //处理获取的数据
                boolean result = dataProcessor.processSnapshot(distroData);
                if (result) {
                    //设置初始化状态
                    distroComponentHolder.findDataStorage(resourceType).finishInitial();
                    return true;
                }
            } catch (Exception e) {
            }
        }
        return false;
    }
}

4、数据更新处理

Rpc 代理对象向集群的节点发送请求获取快照对象，将响应的数据反序列化并缓存更新客户端数据。
ClientSyncData 保存客户端实例所有的属性，namespace，groupName，serviceName，instancePublishInfos。
ConnectionBasedClient 连接数据属性 isNative 等于false-表示非当前Nacos节点维护的客户端连接。

public DistroData getDatumSnapshot(String targetServer) {
    Member member = memberManager.find(targetServer);
    //判断目标节点是否健康
    if (checkTargetServerStatusUnhealthy(member)) {
        throw new DistroException();
    }
    //Distro协议请求
    DistroDataRequest request = new DistroDataRequest();
    //设置数据操作类型
    request.setDataOperation(DataOperation.SNAPSHOT);
    try {
        //Rpc代理对象发送请求
        Response response = clusterRpcClientProxy.sendRequest(member, request,                                       DistroConfig.getInstance().getLoadDataTimeoutMillis());
        //检查响应Code是否成功
        if (checkResponse(response)) {
            //返回响应的数据
            return ((DistroDataResponse) response).getDistroData();
        } else {
            throw new DistroException();
        }
    } catch (NacosException e) {
        throw new DistroException();
    }
}

//DistroClientDataProcessor#processSnapshot
public boolean processSnapshot(DistroData distroData) {
    //将返回数据进行反序列化操作
	ClientSyncDatumSnapshot snapshot = ApplicationUtils.getBean(Serializer.class)
	.deserialize(distroData.getContent(), ClientSyncDatumSnapshot.class);
	for (ClientSyncData each : snapshot.getClientSyncDataList()) {
        //处理连接对象
		handlerClientSyncData(each);
	}
	return true;
}

private void handlerClientSyncData(ClientSyncData clientSyncData) {
    //将连接数据添加到客户端管理器，new ConnectionBasedClient(clientId, false, revision);
	clientManager.syncClientConnected(clientSyncData.getClientId(), clientSyncData.getAttributes());
	Client client = clientManager.getClient(clientSyncData.getClientId());
    //更新连接发布的实例数据
	upgradeClient(client, clientSyncData);
}

private void upgradeClient(Client client, ClientSyncData clientSyncData) {
    //同步的实例
    Set<Service> syncedService = new HashSet<>();
    //处理批量的实例同步
    processBatchInstanceDistroData(syncedService, client, clientSyncData);
    List<String> namespaces = clientSyncData.getNamespaces();
    List<String> groupNames = clientSyncData.getGroupNames();
    List<String> serviceNames = clientSyncData.getServiceNames();
    List<InstancePublishInfo> instances = clientSyncData.getInstancePublishInfos();
    for (int i = 0; i < namespaces.size(); i++) {
        Service service = Service.newService(namespaces.get(i), groupNames.get(i), serviceNames.get(i));
        Service singleton = ServiceManager.getInstance().getSingleton(service);
        syncedService.add(singleton);
        InstancePublishInfo instancePublishInfo = instances.get(i);
        if (!instancePublishInfo.equals(client.getInstancePublishInfo(singleton))) {
           	//客户端添加服务对应的实例数据
            client.addServiceInstance(singleton, instancePublishInfo);
            //发送客户端服务注册事件
            NotifyCenter.publishEvent(
                    new ClientOperationEvent.ClientRegisterServiceEvent(singleton, client.getClientId()));
        }
    }
	//发布的实例在最新同步实例的集合是否存在，不存在则删除客户端的服务数据
    for (Service each : client.getAllPublishedService()) {
        if (!syncedService.contains(each)) {
            //删除不在同步集合的服务和实例
            client.removeServiceInstance(each);
            NotifyCenter.publishEvent(
                    new ClientOperationEvent.ClientDeregisterServiceEvent(each, client.getClientId()));
        }
    }
}

标签

#java #开源 #微服务