Examples with Datum com.alibaba.nacos.naming.consistency.Datum used on opensource projects

Example 21 with Datum

use of com.alibaba.nacos.naming.consistency.Datum in project nacos by alibaba.

the class ServiceManagerTest method testUpdateIpAddresses.

@Test
public void testUpdateIpAddresses() throws Exception {
    List<Instance> instanceList = serviceManager.updateIpAddresses(service, UtilsAndCommons.UPDATE_INSTANCE_ACTION_ADD, true, instance);
    Assert.assertEquals(1, instanceList.size());
    Assert.assertEquals(instance, instanceList.get(0));
    Assert.assertEquals(1, service.getClusterMap().size());
    Assert.assertEquals(new Cluster(instance.getClusterName(), service), service.getClusterMap().get(TEST_CLUSTER_NAME));
    Datum datam = new Datum();
    datam.key = KeyBuilder.buildInstanceListKey(TEST_NAMESPACE, TEST_SERVICE_NAME, true);
    Instances instances = new Instances();
    instanceList.add(instance2);
    instances.setInstanceList(instanceList);
    datam.value = instances;
    when(consistencyService.get(KeyBuilder.buildInstanceListKey(TEST_NAMESPACE, TEST_SERVICE_NAME, true))).thenReturn(datam);
    service.getClusterMap().get(TEST_CLUSTER_NAME).updateIps(instanceList, true);
    instanceList = serviceManager.updateIpAddresses(service, UtilsAndCommons.UPDATE_INSTANCE_ACTION_REMOVE, true, instance);
    Assert.assertEquals(1, instanceList.size());
    Assert.assertEquals(instance2, instanceList.get(0));
    Assert.assertEquals(1, service.getClusterMap().size());
    Assert.assertEquals(new Cluster(instance.getClusterName(), service), service.getClusterMap().get(TEST_CLUSTER_NAME));
}

Example 22 with Datum

use of com.alibaba.nacos.naming.consistency.Datum in project nacos by alibaba.

the class JacksonSerializerTest method testDeserializeMap.

@Test
@SuppressWarnings("checkstyle:linelength")
public void testDeserializeMap() {
    String example = "{\"datum\":{\"key\":\"instances\",\"value\":{\"instanceList\":[{\"ip\":\"1.1.1.1\",\"port\":1234,\"weight\":1.0,\"healthy\":true,\"enabled\":true,\"ephemeral\":true,\"clusterName\":\"cluster\",\"metadata\":{},\"lastBeat\":1590563397533,\"marked\":false,\"instanceIdGenerator\":\"simple\",\"instanceHeartBeatInterval\":5000,\"instanceHeartBeatTimeOut\":15000,\"ipDeleteTimeout\":30000}]},\"timestamp\":100000}}";
    Map<String, Datum<Instances>> actual = serializer.deserializeMap(ByteUtils.toBytes(example), Instances.class);
    assertEquals(actual.size(), 1);
    assertTrue(actual.containsKey("datum"));
    Datum<Instances> actualDatum = actual.get("datum");
    assertEquals("instances", actualDatum.key);
    assertEquals(100000L, actualDatum.timestamp.get());
    assertEquals(1, actualDatum.value.getInstanceList().size());
    Instance actualInstance = actualDatum.value.getInstanceList().get(0);
    assertEquals("1.1.1.1", actualInstance.getIp());
    assertEquals("cluster", actualInstance.getClusterName());
    assertEquals(1234, actualInstance.getPort());
}

Example 23 with Datum

use of com.alibaba.nacos.naming.consistency.Datum in project nacos by alibaba.

the class RaftCore method signalDelete.

/**
 * Signal delete record. If not leader, signal leader delete. If leader, try to commit delete.
 *
 * @param key key
 * @throws Exception any exception during delete
 */
public void signalDelete(final String key) throws Exception {
    if (stopWork) {
        throw new IllegalStateException("old raft protocol already stop work");
    }
    OPERATE_LOCK.lock();
    try {
        if (!isLeader()) {
            Map<String, String> params = new HashMap<>(1);
            params.put("key", URLEncoder.encode(key, "UTF-8"));
            raftProxy.proxy(getLeader().ip, API_DEL, params, HttpMethod.DELETE);
            return;
        }
        // construct datum:
        Datum datum = new Datum();
        datum.key = key;
        ObjectNode json = JacksonUtils.createEmptyJsonNode();
        json.replace("datum", JacksonUtils.transferToJsonNode(datum));
        json.replace("source", JacksonUtils.transferToJsonNode(peers.local()));
        onDelete(datum.key, peers.local());
        for (final String server : peers.allServersWithoutMySelf()) {
            String url = buildUrl(server, API_ON_DEL);
            HttpClient.asyncHttpDeleteLarge(url, null, json.toString(), new Callback<String>() {

                @Override
                public void onReceive(RestResult<String> result) {
                    if (!result.ok()) {
                        Loggers.RAFT.warn("[RAFT] failed to delete data from peer, datumId={}, peer={}, http code={}", key, server, result.getCode());
                        return;
                    }
                    RaftPeer local = peers.local();
                    local.resetLeaderDue();
                }

                @Override
                public void onError(Throwable throwable) {
                    Loggers.RAFT.error("[RAFT] failed to delete data from peer", throwable);
                }

                @Override
                public void onCancel() {
                }
            });
        }
    } finally {
        OPERATE_LOCK.unlock();
    }
}

Example 24 with Datum

use of com.alibaba.nacos.naming.consistency.Datum in project nacos by alibaba.

the class RaftCore method receivedBeat.

/**
 * Received beat from leader. // TODO split method to multiple smaller method.
 *
 * @param beat beat information from leader
 * @return self-peer information
 * @throws Exception any exception during handle
 */
public RaftPeer receivedBeat(JsonNode beat) throws Exception {
    if (stopWork) {
        throw new IllegalStateException("old raft protocol already stop work");
    }
    final RaftPeer local = peers.local();
    final RaftPeer remote = new RaftPeer();
    JsonNode peer = beat.get("peer");
    remote.ip = peer.get("ip").asText();
    remote.state = RaftPeer.State.valueOf(peer.get("state").asText());
    remote.term.set(peer.get("term").asLong());
    remote.heartbeatDueMs = peer.get("heartbeatDueMs").asLong();
    remote.leaderDueMs = peer.get("leaderDueMs").asLong();
    remote.voteFor = peer.get("voteFor").asText();
    if (remote.state != RaftPeer.State.LEADER) {
        Loggers.RAFT.info("[RAFT] invalid state from master, state: {}, remote peer: {}", remote.state, JacksonUtils.toJson(remote));
        throw new IllegalArgumentException("invalid state from master, state: " + remote.state);
    }
    if (local.term.get() > remote.term.get()) {
        Loggers.RAFT.info("[RAFT] out of date beat, beat-from-term: {}, beat-to-term: {}, remote peer: {}, and leaderDueMs: {}", remote.term.get(), local.term.get(), JacksonUtils.toJson(remote), local.leaderDueMs);
        throw new IllegalArgumentException("out of date beat, beat-from-term: " + remote.term.get() + ", beat-to-term: " + local.term.get());
    }
    if (local.state != RaftPeer.State.FOLLOWER) {
        Loggers.RAFT.info("[RAFT] make remote as leader, remote peer: {}", JacksonUtils.toJson(remote));
        // mk follower
        local.state = RaftPeer.State.FOLLOWER;
        local.voteFor = remote.ip;
    }
    final JsonNode beatDatums = beat.get("datums");
    local.resetLeaderDue();
    local.resetHeartbeatDue();
    peers.makeLeader(remote);
    if (!switchDomain.isSendBeatOnly()) {
        Map<String, Integer> receivedKeysMap = new HashMap<>(datums.size());
        for (Map.Entry<String, Datum> entry : datums.entrySet()) {
            receivedKeysMap.put(entry.getKey(), 0);
        }
        // now check datums
        List<String> batch = new ArrayList<>();
        int processedCount = 0;
        if (Loggers.RAFT.isDebugEnabled()) {
            Loggers.RAFT.debug("[RAFT] received beat with {} keys, RaftCore.datums' size is {}, remote server: {}, term: {}, local term: {}", beatDatums.size(), datums.size(), remote.ip, remote.term, local.term);
        }
        for (Object object : beatDatums) {
            processedCount = processedCount + 1;
            JsonNode entry = (JsonNode) object;
            String key = entry.get("key").asText();
            final String datumKey;
            if (KeyBuilder.matchServiceMetaKey(key)) {
                datumKey = KeyBuilder.detailServiceMetaKey(key);
            } else if (KeyBuilder.matchInstanceListKey(key)) {
                datumKey = KeyBuilder.detailInstanceListkey(key);
            } else {
                // ignore corrupted key:
                continue;
            }
            long timestamp = entry.get("timestamp").asLong();
            receivedKeysMap.put(datumKey, 1);
            try {
                if (datums.containsKey(datumKey) && datums.get(datumKey).timestamp.get() >= timestamp && processedCount < beatDatums.size()) {
                    continue;
                }
                if (!(datums.containsKey(datumKey) && datums.get(datumKey).timestamp.get() >= timestamp)) {
                    batch.add(datumKey);
                }
                if (batch.size() < 50 && processedCount < beatDatums.size()) {
                    continue;
                }
                String keys = StringUtils.join(batch, ",");
                if (batch.size() <= 0) {
                    continue;
                }
                Loggers.RAFT.info("get datums from leader: {}, batch size is {}, processedCount is {}" + ", datums' size is {}, RaftCore.datums' size is {}", getLeader().ip, batch.size(), processedCount, beatDatums.size(), datums.size());
                // update datum entry
                String url = buildUrl(remote.ip, API_GET);
                Map<String, String> queryParam = new HashMap<>(1);
                queryParam.put("keys", URLEncoder.encode(keys, "UTF-8"));
                HttpClient.asyncHttpGet(url, null, queryParam, new Callback<String>() {

                    @Override
                    public void onReceive(RestResult<String> result) {
                        if (!result.ok()) {
                            return;
                        }
                        List<JsonNode> datumList = JacksonUtils.toObj(result.getData(), new TypeReference<List<JsonNode>>() {
                        });
                        for (JsonNode datumJson : datumList) {
                            Datum newDatum = null;
                            OPERATE_LOCK.lock();
                            try {
                                Datum oldDatum = getDatum(datumJson.get("key").asText());
                                if (oldDatum != null && datumJson.get("timestamp").asLong() <= oldDatum.timestamp.get()) {
                                    Loggers.RAFT.info("[NACOS-RAFT] timestamp is smaller than that of mine, key: {}, remote: {}, local: {}", datumJson.get("key").asText(), datumJson.get("timestamp").asLong(), oldDatum.timestamp);
                                    continue;
                                }
                                if (KeyBuilder.matchServiceMetaKey(datumJson.get("key").asText())) {
                                    Datum<Service> serviceDatum = new Datum<>();
                                    serviceDatum.key = datumJson.get("key").asText();
                                    serviceDatum.timestamp.set(datumJson.get("timestamp").asLong());
                                    serviceDatum.value = JacksonUtils.toObj(datumJson.get("value").toString(), Service.class);
                                    newDatum = serviceDatum;
                                }
                                if (KeyBuilder.matchInstanceListKey(datumJson.get("key").asText())) {
                                    Datum<Instances> instancesDatum = new Datum<>();
                                    instancesDatum.key = datumJson.get("key").asText();
                                    instancesDatum.timestamp.set(datumJson.get("timestamp").asLong());
                                    instancesDatum.value = JacksonUtils.toObj(datumJson.get("value").toString(), Instances.class);
                                    newDatum = instancesDatum;
                                }
                                if (newDatum == null || newDatum.value == null) {
                                    Loggers.RAFT.error("receive null datum: {}", datumJson);
                                    continue;
                                }
                                raftStore.write(newDatum);
                                datums.put(newDatum.key, newDatum);
                                notifier.notify(newDatum.key, DataOperation.CHANGE, newDatum.value);
                                local.resetLeaderDue();
                                if (local.term.get() + 100 > remote.term.get()) {
                                    getLeader().term.set(remote.term.get());
                                    local.term.set(getLeader().term.get());
                                } else {
                                    local.term.addAndGet(100);
                                }
                                raftStore.updateTerm(local.term.get());
                                Loggers.RAFT.info("data updated, key: {}, timestamp: {}, from {}, local term: {}", newDatum.key, newDatum.timestamp, JacksonUtils.toJson(remote), local.term);
                            } catch (Throwable e) {
                                Loggers.RAFT.error("[RAFT-BEAT] failed to sync datum from leader, datum: {}", newDatum, e);
                            } finally {
                                OPERATE_LOCK.unlock();
                            }
                        }
                        try {
                            TimeUnit.MILLISECONDS.sleep(200);
                        } catch (InterruptedException e) {
                            Loggers.RAFT.error("[RAFT-BEAT] Interrupted error ", e);
                        }
                        return;
                    }

                    @Override
                    public void onError(Throwable throwable) {
                        Loggers.RAFT.error("[RAFT-BEAT] failed to sync datum from leader", throwable);
                    }

                    @Override
                    public void onCancel() {
                    }
                });
                batch.clear();
            } catch (Exception e) {
                Loggers.RAFT.error("[NACOS-RAFT] failed to handle beat entry, key: {}", datumKey);
            }
        }
        List<String> deadKeys = new ArrayList<>();
        for (Map.Entry<String, Integer> entry : receivedKeysMap.entrySet()) {
            if (entry.getValue() == 0) {
                deadKeys.add(entry.getKey());
            }
        }
        for (String deadKey : deadKeys) {
            try {
                deleteDatum(deadKey);
            } catch (Exception e) {
                Loggers.RAFT.error("[NACOS-RAFT] failed to remove entry, key={} {}", deadKey, e);
            }
        }
    }
    return local;
}

Example 25 with Datum

use of com.alibaba.nacos.naming.consistency.Datum in project nacos by alibaba.

the class RaftCore method signalPublish.

/**
 * Signal publish new record. If not leader, signal to leader. If leader, try to commit publish.
 *
 * @param key   key
 * @param value value
 * @throws Exception any exception during publish
 */
public void signalPublish(String key, Record value) throws Exception {
    if (stopWork) {
        throw new IllegalStateException("old raft protocol already stop work");
    }
    if (!isLeader()) {
        ObjectNode params = JacksonUtils.createEmptyJsonNode();
        params.put("key", key);
        params.replace("value", JacksonUtils.transferToJsonNode(value));
        Map<String, String> parameters = new HashMap<>(1);
        parameters.put("key", key);
        final RaftPeer leader = getLeader();
        raftProxy.proxyPostLarge(leader.ip, API_PUB, params.toString(), parameters);
        return;
    }
    OPERATE_LOCK.lock();
    try {
        final long start = System.currentTimeMillis();
        final Datum datum = new Datum();
        datum.key = key;
        datum.value = value;
        if (getDatum(key) == null) {
            datum.timestamp.set(1L);
        } else {
            datum.timestamp.set(getDatum(key).timestamp.incrementAndGet());
        }
        ObjectNode json = JacksonUtils.createEmptyJsonNode();
        json.replace("datum", JacksonUtils.transferToJsonNode(datum));
        json.replace("source", JacksonUtils.transferToJsonNode(peers.local()));
        onPublish(datum, peers.local());
        final String content = json.toString();
        final CountDownLatch latch = new CountDownLatch(peers.majorityCount());
        for (final String server : peers.allServersIncludeMyself()) {
            if (isLeader(server)) {
                latch.countDown();
                continue;
            }
            final String url = buildUrl(server, API_ON_PUB);
            HttpClient.asyncHttpPostLarge(url, Arrays.asList("key", key), content, new Callback<String>() {

                @Override
                public void onReceive(RestResult<String> result) {
                    if (!result.ok()) {
                        Loggers.RAFT.warn("[RAFT] failed to publish data to peer, datumId={}, peer={}, http code={}", datum.key, server, result.getCode());
                        return;
                    }
                    latch.countDown();
                }

                @Override
                public void onError(Throwable throwable) {
                    Loggers.RAFT.error("[RAFT] failed to publish data to peer", throwable);
                }

                @Override
                public void onCancel() {
                }
            });
        }
        if (!latch.await(UtilsAndCommons.RAFT_PUBLISH_TIMEOUT, TimeUnit.MILLISECONDS)) {
            // only majority servers return success can we consider this update success
            Loggers.RAFT.error("data publish failed, caused failed to notify majority, key={}", key);
            throw new IllegalStateException("data publish failed, caused failed to notify majority, key=" + key);
        }
        long end = System.currentTimeMillis();
        Loggers.RAFT.info("signalPublish cost {} ms, key: {}", (end - start), key);
    } finally {
        OPERATE_LOCK.unlock();
    }
}