Examples with StoreKey com.github.ambry.store.StoreKey used on opensource projects

Example 71 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTest method replicaThreadTestConverter.

/**
 * Tests that replication between a local and remote server who have different
 * blob IDs for the same blobs (via StoreKeyConverter)
 * @throws Exception
 */
@Test
public void replicaThreadTestConverter() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockHost expectedLocalHost = new MockHost(localHost.dataNodeId, clusterMap);
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition = new HashMap<>();
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    int batchSize = 4;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, transformer, null, null);
    Map<DataNodeId, List<RemoteReplicaInfo>> replicasToReplicate = replicasAndThread.getFirst();
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    /*
        STORE KEY CONVERTER MAPPING
        Key     Value
        B0      B0'
        B1      B1'
        B2      null

        BEFORE
        Local   Remote
        B0'     B0
                B1
                B2

        AFTER
        Local   Remote
        B0'     B0
        B1'     B1
                B2
        B0 is B0' for local,
        B1 is B1' for local,
        B2 is null for local,
        so it already has B0/B0'
        B1 is transferred to B1'
        and B2 is invalid for L
        so it does not count as missing
        Missing Keys: 1
    */
    Map<PartitionId, List<BlobId>> partitionIdToDeleteBlobId = new HashMap<>();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    Map<PartitionId, BlobId> expectedPartitionIdToDeleteBlobId = new HashMap<>();
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        List<BlobId> deleteBlobIds = new ArrayList<>();
        partitionIdToDeleteBlobId.put(partitionId, deleteBlobIds);
        BlobId b0 = generateRandomBlobId(partitionId);
        deleteBlobIds.add(b0);
        BlobId b0p = generateRandomBlobId(partitionId);
        expectedPartitionIdToDeleteBlobId.put(partitionId, b0p);
        BlobId b1 = generateRandomBlobId(partitionId);
        BlobId b1p = generateRandomBlobId(partitionId);
        BlobId b2 = generateRandomBlobId(partitionId);
        deleteBlobIds.add(b2);
        conversionMap.put(b0, b0p);
        conversionMap.put(b1, b1p);
        conversionMap.put(b2, null);
        // Convert current conversion map so that BlobIdTransformer can
        // create b1p in expectedLocalHost
        storeKeyConverter.setConversionMap(conversionMap);
        storeKeyConverter.convert(conversionMap.keySet());
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0p), Arrays.asList(localHost));
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0, b1, b2), Arrays.asList(remoteHost));
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0p, b1), Arrays.asList(null, transformer), Arrays.asList(expectedLocalHost));
        // Check that expected local host contains the correct blob ids
        Set<BlobId> expectedLocalHostBlobIds = new HashSet<>();
        expectedLocalHostBlobIds.add(b0p);
        expectedLocalHostBlobIds.add(b1p);
        for (MessageInfo messageInfo : expectedLocalHost.infosByPartition.get(partitionId)) {
            assertTrue("Remove should never fail", expectedLocalHostBlobIds.remove(messageInfo.getStoreKey()));
        }
        assertTrue("expectedLocalHostBlobIds should now be empty", expectedLocalHostBlobIds.isEmpty());
    }
    storeKeyConverter.setConversionMap(conversionMap);
    int expectedIndex = assertMissingKeysAndFixMissingStoreKeys(0, 2, batchSize, 1, replicaThread, remoteHost, replicasToReplicate);
    // Check that there are no missing buffers between expectedLocalHost and LocalHost
    Map<PartitionId, List<ByteBuffer>> missingBuffers = expectedLocalHost.getMissingBuffers(localHost.buffersByPartition);
    assertTrue(missingBuffers.isEmpty());
    missingBuffers = localHost.getMissingBuffers(expectedLocalHost.buffersByPartition);
    assertTrue(missingBuffers.isEmpty());
    // delete blob
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        List<BlobId> deleteBlobIds = partitionIdToDeleteBlobId.get(partitionId);
        for (BlobId deleteBlobId : deleteBlobIds) {
            addDeleteMessagesToReplicasOfPartition(partitionId, deleteBlobId, Arrays.asList(remoteHost));
        }
        addDeleteMessagesToReplicasOfPartition(partitionId, expectedPartitionIdToDeleteBlobId.get(partitionId), Arrays.asList(expectedLocalHost));
    }
    expectedIndex = assertMissingKeysAndFixMissingStoreKeys(expectedIndex, 2, batchSize, 0, replicaThread, remoteHost, replicasToReplicate);
    // Check that there are no missing buffers between expectedLocalHost and LocalHost
    missingBuffers = expectedLocalHost.getMissingBuffers(localHost.buffersByPartition);
    assertTrue(missingBuffers.isEmpty());
    missingBuffers = localHost.getMissingBuffers(expectedLocalHost.buffersByPartition);
    assertTrue(missingBuffers.isEmpty());
    // 3 unconverted + 2 unconverted deleted expected missing buffers
    verifyNoMoreMissingKeysAndExpectedMissingBufferCount(remoteHost, localHost, replicaThread, replicasToReplicate, idsToBeIgnoredByPartition, storeKeyConverter, expectedIndex, expectedIndex, 5);
}

Example 72 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTest method replicaThreadLifeVersionLocalLessThanRemote_MissingPuts.

/**
 * Tests when the local store missing put records with lifeVersion greater than 0
 */
@Test
public void replicaThreadLifeVersionLocalLessThanRemote_MissingPuts() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    storeKeyConverter.setConversionMap(conversionMap);
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition = new HashMap<>();
    Map<PartitionId, List<StoreKey>> idsToBeTtlUpdatedByPartition = new HashMap<>();
    short lifeVersion = 1;
    for (int i = 0; i < partitionIds.size(); i++) {
        List<StoreKey> toBeIgnored = new ArrayList<>();
        List<StoreKey> toBeTtlUpdated = new ArrayList<>();
        PartitionId partitionId = partitionIds.get(i);
        // Adding 1 put to remoteHost at lifeVersion 0
        List<StoreKey> ids = addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 1);
        // Adding 1 put to remoteHost at lifeVersion 1
        ids.addAll(addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), lifeVersion, 1));
        // Adding one put to remoteHost at lifeVersion 1, which would be ttl updated later at lifeVersion 1
        StoreKey toTtlUpdateId = addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), lifeVersion, 1).get(0);
        ids.add(toTtlUpdateId);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, toTtlUpdateId, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, lifeVersion);
        toBeTtlUpdated.add(toTtlUpdateId);
        // Adding one put to remoteHost at lifeVersion 0, which would be ttl updated later at lifeVersion 1
        toTtlUpdateId = addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 1).get(0);
        ids.add(toTtlUpdateId);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, toTtlUpdateId, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, lifeVersion);
        toBeTtlUpdated.add(toTtlUpdateId);
        // Adding one put to remoteHost, which would be deleted later
        StoreKey toDeleteId = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost), lifeVersion, 1).get(0);
        ids.add(toDeleteId);
        addDeleteMessagesToReplicasOfPartition(partitionId, toDeleteId, Collections.singletonList(remoteHost), lifeVersion, EXPIRY_TIME_MS);
        toBeIgnored.add(toDeleteId);
        // Adding one put to remoteHost, which would be ttl updated and deleted later
        StoreKey toDeleteAndTtlUpdateId = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost), lifeVersion, 1).get(0);
        ids.add(toDeleteAndTtlUpdateId);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, toDeleteAndTtlUpdateId, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, lifeVersion);
        toBeTtlUpdated.add(toDeleteAndTtlUpdateId);
        addDeleteMessagesToReplicasOfPartition(partitionId, toDeleteAndTtlUpdateId, Collections.singletonList(remoteHost), lifeVersion, UPDATED_EXPIRY_TIME_MS);
        toBeIgnored.add(toDeleteAndTtlUpdateId);
        // Adding one put to remoteHost at lifeVersion 0, delete it and then add undelete at lifeVersion 1
        StoreKey deleteAndUndeleteId = addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 1).get(0);
        ids.add(deleteAndUndeleteId);
        addDeleteMessagesToReplicasOfPartition(partitionId, deleteAndUndeleteId, Collections.singletonList(remoteHost), (short) 0, EXPIRY_TIME_MS);
        addUndeleteMessagesToReplicasOfPartition(partitionId, deleteAndUndeleteId, Collections.singletonList(remoteHost), lifeVersion);
        idsToBeIgnoredByPartition.put(partitionId, toBeIgnored);
        idsToBeTtlUpdatedByPartition.put(partitionId, toBeTtlUpdated);
        // Adding one put to both remote and local host.
        ids.addAll(addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(localHost, remoteHost), lifeVersion, 1));
    }
    int batchSize = 100;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, transformer, null, null);
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasAndThread.getFirst().get(remoteHost.dataNodeId);
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    int missingKeyCount = 5;
    List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
    for (int i = 0; i < response.size(); i++) {
        assertEquals(missingKeyCount, response.get(i).missingStoreMessages.size());
        remoteReplicaInfos.get(i).setToken(response.get(i).remoteToken);
    }
    replicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos, response, false);
    for (int i = 0; i < response.size(); i++) {
        assertEquals("Token should have been set correctly in fixMissingStoreKeys()", response.get(i).remoteToken, remoteReplicaInfos.get(i).getToken());
    }
    // Don't compare buffers here, PutBuffer might be different since we might change the lifeVersion.
    for (Map.Entry<PartitionId, List<MessageInfo>> localInfoEntry : localHost.infosByPartition.entrySet()) {
        assertEquals("MessageInfo number mismatch", 8, localInfoEntry.getValue().size());
    }
    checkBlobMessagesAreEqualInLocalAndRemoteHosts(localHost, remoteHost, idsToBeIgnoredByPartition, idsToBeTtlUpdatedByPartition);
}

Example 73 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTest method replicaThreadTest.

/**
 * Tests {@link ReplicaThread#exchangeMetadata(ConnectedChannel, List)} and
 * {@link ReplicaThread#fixMissingStoreKeys(ConnectedChannel, List, List, boolean)} for valid puts, deletes, expired keys and
 * corrupt blobs.
 * @throws Exception
 */
@Test
public void replicaThreadTest() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition = new HashMap<>();
    for (int i = 0; i < partitionIds.size(); i++) {
        List<StoreKey> idsToBeIgnored = new ArrayList<>();
        PartitionId partitionId = partitionIds.get(i);
        // add 6 messages to both hosts.
        StoreKey toDeleteId = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(localHost, remoteHost), 6).get(0);
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        boolean toEncrypt = TestUtils.RANDOM.nextBoolean();
        // add an expired message to the remote host only
        StoreKey id = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, accountId, containerId, partitionId, toEncrypt, BlobId.BlobDataType.DATACHUNK);
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, accountId, containerId, toEncrypt);
        remoteHost.addMessage(partitionId, new MessageInfo(id, msgInfoAndBuffer.byteBuffer.remaining(), 1, accountId, containerId, msgInfoAndBuffer.messageInfo.getOperationTimeMs()), msgInfoAndBuffer.byteBuffer);
        idsToBeIgnored.add(id);
        // add 3 messages to the remote host only
        addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 3);
        accountId = Utils.getRandomShort(TestUtils.RANDOM);
        containerId = Utils.getRandomShort(TestUtils.RANDOM);
        toEncrypt = TestUtils.RANDOM.nextBoolean();
        // add a corrupt message to the remote host only
        id = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, accountId, containerId, partitionId, toEncrypt, BlobId.BlobDataType.DATACHUNK);
        msgInfoAndBuffer = createPutMessage(id, accountId, containerId, toEncrypt);
        byte[] data = msgInfoAndBuffer.byteBuffer.array();
        // flip every bit in the array
        for (int j = 0; j < data.length; j++) {
            data[j] ^= 0xFF;
        }
        remoteHost.addMessage(partitionId, msgInfoAndBuffer.messageInfo, msgInfoAndBuffer.byteBuffer);
        idsToBeIgnored.add(id);
        // add 3 messages to the remote host only
        addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 3);
        // add delete record for the very first blob in the remote host only
        addDeleteMessagesToReplicasOfPartition(partitionId, toDeleteId, Collections.singletonList(remoteHost));
        // PUT and DELETE a blob in the remote host only
        id = addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), 1).get(0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost));
        idsToBeIgnored.add(id);
        // add 2 or 3 messages (depending on whether partition is even-numbered or odd-numbered) to the remote host only
        addPutMessagesToReplicasOfPartition(partitionId, Collections.singletonList(remoteHost), i % 2 == 0 ? 2 : 3);
        idsToBeIgnoredByPartition.put(partitionId, idsToBeIgnored);
        // ensure that the first key is not deleted in the local host
        assertNull(toDeleteId + " should not be deleted in the local host", getMessageInfo(toDeleteId, localHost.infosByPartition.get(partitionId), true, false, false));
    }
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    int batchSize = 4;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, transformer, null, null);
    Map<DataNodeId, List<RemoteReplicaInfo>> replicasToReplicate = replicasAndThread.getFirst();
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    Map<PartitionId, List<ByteBuffer>> missingBuffers = remoteHost.getMissingBuffers(localHost.buffersByPartition);
    for (Map.Entry<PartitionId, List<ByteBuffer>> entry : missingBuffers.entrySet()) {
        if (partitionIds.indexOf(entry.getKey()) % 2 == 0) {
            assertEquals("Missing buffers count mismatch", 13, entry.getValue().size());
        } else {
            assertEquals("Missing buffers count mismatch", 14, entry.getValue().size());
        }
    }
    // 1st and 2nd iterations - no keys missing because all data is in both hosts
    // 3rd iteration - 3 missing keys (one expired)
    // 4th iteration - 3 missing keys (one expired) - the corrupt key also shows up as missing but is ignored later
    // 5th iteration - 1 missing key (1 key from prev cycle, 1 deleted key, 1 never present key but deleted in remote)
    // 6th iteration - 2 missing keys (2 entries i.e put,delete of never present key)
    int[] missingKeysCounts = { 0, 0, 3, 3, 1, 2 };
    int[] missingBuffersCount = { 12, 12, 9, 7, 6, 4 };
    int expectedIndex = 0;
    int missingBuffersIndex = 0;
    for (int missingKeysCount : missingKeysCounts) {
        expectedIndex = assertMissingKeysAndFixMissingStoreKeys(expectedIndex, batchSize - 1, batchSize, missingKeysCount, replicaThread, remoteHost, replicasToReplicate);
        missingBuffers = remoteHost.getMissingBuffers(localHost.buffersByPartition);
        for (Map.Entry<PartitionId, List<ByteBuffer>> entry : missingBuffers.entrySet()) {
            if (partitionIds.indexOf(entry.getKey()) % 2 == 0) {
                assertEquals("Missing buffers count mismatch for iteration count " + missingBuffersIndex, missingBuffersCount[missingBuffersIndex], entry.getValue().size());
            } else {
                assertEquals("Missing buffers count mismatch for iteration count " + missingBuffersIndex, missingBuffersCount[missingBuffersIndex] + 1, entry.getValue().size());
            }
        }
        missingBuffersIndex++;
    }
    // Test the case where some partitions have missing keys, but not all.
    List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), replicasToReplicate.get(remoteHost.dataNodeId));
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasToReplicate.get(remoteHost.dataNodeId);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
    for (int i = 0; i < response.size(); i++) {
        if (i % 2 == 0) {
            assertEquals(0, response.get(i).missingStoreMessages.size());
            assertEquals(expectedIndex, ((MockFindToken) response.get(i).remoteToken).getIndex());
        } else {
            assertEquals(1, response.get(i).missingStoreMessages.size());
            assertEquals(expectedIndex + 1, ((MockFindToken) response.get(i).remoteToken).getIndex());
        }
    }
    replicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHost, batchSize), replicasToReplicate.get(remoteHost.dataNodeId), response, false);
    for (int i = 0; i < response.size(); i++) {
        assertEquals("Token should have been set correctly in fixMissingStoreKeys()", response.get(i).remoteToken, replicasToReplicate.get(remoteHost.dataNodeId).get(i).getToken());
    }
    // 1 expired + 1 corrupt + 1 put (never present) + 1 deleted (never present) expected missing buffers
    verifyNoMoreMissingKeysAndExpectedMissingBufferCount(remoteHost, localHost, replicaThread, replicasToReplicate, idsToBeIgnoredByPartition, storeKeyConverter, expectedIndex, expectedIndex + 1, 4);
}

Example 74 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTest method ttlUpdateReplicationTest.

/**
 * Tests replication of TTL updates
 * @throws Exception
 */
@Test
public void ttlUpdateReplicationTest() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockHost expectedLocalHost = new MockHost(localHost.dataNodeId, clusterMap);
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    storeKeyConverter.setConversionMap(conversionMap);
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    int numMessagesInEachPart = 0;
    Map<PartitionId, StoreKey> idsDeletedLocallyByPartition = new HashMap<>();
    List<MockHost> remoteHostOnly = Collections.singletonList(remoteHost);
    List<MockHost> expectedLocalHostOnly = Collections.singletonList(expectedLocalHost);
    List<MockHost> localHostAndExpectedLocalHost = Arrays.asList(localHost, expectedLocalHost);
    List<MockHost> remoteHostAndExpectedLocalHost = Arrays.asList(remoteHost, expectedLocalHost);
    List<MockHost> allHosts = Arrays.asList(localHost, expectedLocalHost, remoteHost);
    for (PartitionId pid : partitionIds) {
        // add 3 put messages to both hosts (also add to expectedLocal)
        List<StoreKey> ids = addPutMessagesToReplicasOfPartition(pid, allHosts, 3);
        // delete 1 of the messages in the local host only
        addDeleteMessagesToReplicasOfPartition(pid, ids.get(0), localHostAndExpectedLocalHost);
        idsDeletedLocallyByPartition.put(pid, ids.get(0));
        // ttl update 1 of the messages in the local host only
        addTtlUpdateMessagesToReplicasOfPartition(pid, ids.get(1), localHostAndExpectedLocalHost, UPDATED_EXPIRY_TIME_MS);
        // remote host only
        // add 2 put messages
        ids.addAll(addPutMessagesToReplicasOfPartition(pid, remoteHostOnly, 1));
        ids.addAll(addPutMessagesToReplicasOfPartition(pid, remoteHostAndExpectedLocalHost, 1));
        // ttl update all 5 put messages
        for (int i = ids.size() - 1; i >= 0; i--) {
            List<MockHost> hostList = remoteHostOnly;
            if (i == 2 || i == 4) {
                hostList = remoteHostAndExpectedLocalHost;
            }
            // doing it in reverse order so that a put and ttl update arrive in the same batch
            addTtlUpdateMessagesToReplicasOfPartition(pid, ids.get(i), hostList, UPDATED_EXPIRY_TIME_MS);
        }
        // delete one of the keys that has put and ttl update on local host
        addDeleteMessagesToReplicasOfPartition(pid, ids.get(1), remoteHostAndExpectedLocalHost);
        // delete one of the keys that has put and ttl update on remote only
        addDeleteMessagesToReplicasOfPartition(pid, ids.get(3), remoteHostOnly);
        // add a TTL update and delete message without a put msg (compaction can create such a situation)
        BlobId id = generateRandomBlobId(pid);
        addTtlUpdateMessagesToReplicasOfPartition(pid, id, remoteHostOnly, UPDATED_EXPIRY_TIME_MS);
        addDeleteMessagesToReplicasOfPartition(pid, id, remoteHostOnly);
        // message transformation test cases
        // a blob ID with PUT and TTL update in both remote and local
        BlobId b0 = generateRandomBlobId(pid);
        BlobId b0p = generateRandomBlobId(pid);
        // a blob ID with a PUT in the local and PUT and TTL update in remote (with mapping)
        BlobId b1 = generateRandomBlobId(pid);
        BlobId b1p = generateRandomBlobId(pid);
        // a blob ID with PUT and TTL update in remote only (with mapping)
        BlobId b2 = generateRandomBlobId(pid);
        BlobId b2p = generateRandomBlobId(pid);
        // a blob ID with PUT and TTL update in remote (no mapping)
        BlobId b3 = generateRandomBlobId(pid);
        conversionMap.put(b0, b0p);
        conversionMap.put(b1, b1p);
        conversionMap.put(b2, b2p);
        conversionMap.put(b3, null);
        storeKeyConverter.convert(conversionMap.keySet());
        // add as required on local, remote and expected local
        // only PUT of b0p and b1p on local
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0p, b1p), localHostAndExpectedLocalHost);
        // PUT of b0,b1,b2,b3 on remote
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0, b1, b2, b3), remoteHostOnly);
        // PUT of b0, b1, b2 expected in local at the end
        addPutMessagesToReplicasOfPartition(Collections.singletonList(b2), Collections.singletonList(transformer), expectedLocalHostOnly);
        // TTL update of b0 on all hosts
        addTtlUpdateMessagesToReplicasOfPartition(pid, b0p, localHostAndExpectedLocalHost, UPDATED_EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(pid, b0, remoteHostOnly, UPDATED_EXPIRY_TIME_MS);
        // TTL update on b1, b2 and b3 on remote
        addTtlUpdateMessagesToReplicasOfPartition(pid, b1, remoteHostOnly, UPDATED_EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(pid, b1p, expectedLocalHostOnly, UPDATED_EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(pid, b2, remoteHostOnly, UPDATED_EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(pid, b2p, expectedLocalHostOnly, UPDATED_EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(pid, b3, remoteHostOnly, UPDATED_EXPIRY_TIME_MS);
        numMessagesInEachPart = remoteHost.infosByPartition.get(pid).size();
    }
    // After the for loop above, we have records in hosts just like below
    // L|id0|id1|id2|id0D|id1T|   |   |    |    |    |    |    |    |    |   |   |b0p|b1p|   |  |b0pT|    |    |    |
    // R|id0|id1|id2|    |    |id3|id4|id4T|id3T|id2T|id1T|id0T|id1D|id3D|idT|idD|b0 |b1 |b2 |b3|b0T |b1T |b2T | b3T|
    // E|id0|id1|id2|id0D|id1T|   |id4|id4T|    |id2T|    |    |id1D|    |   |   |b0p|b1p|b2p|  |b0pT|b1pT|b2pT|    |
    // 
    // converter map: b0->b0p, b1->b1p, b2->b2p, b3->null
    int batchSize = 4;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, transformer, null, null);
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasAndThread.getFirst().get(remoteHost.dataNodeId);
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    Map<PartitionId, List<ByteBuffer>> missingBuffers = expectedLocalHost.getMissingBuffers(localHost.buffersByPartition);
    // We can see from the table in the comments above, Local has 7 records less than expected local.
    for (Map.Entry<PartitionId, List<ByteBuffer>> entry : missingBuffers.entrySet()) {
        assertEquals("Missing buffers count mismatch", 7, entry.getValue().size());
    }
    // 1st iteration - 0 missing keys (3 puts already present, one put missing but del in remote, 1 ttl update will be
    // applied, 1 delete will be applied): Remote returns: id0T, id1TD, id2T, id3TD. id3 put missing, but it's deleted.
    // id1 apply delete, id2 apply ttl update. Token index is pointing to id3.
    // 2nd iteration - 1 missing key, 1 of which will also be ttl updated (one key with put + ttl update missing but
    // del in remote, one put and ttl update replicated): Remote returns: id3TD, id4T. id4 put missing, id3 deleted.
    // Token index is pointing to id3T.
    // 3rd iteration - 0 missing keys (1 ttl update missing but del in remote, 1 already ttl updated in iter 1, 1 key
    // already ttl updated in local, 1 key del local): Remote returns: id3TD, id2T, id1TD, id0T. Token index is pointing
    // to id0T.
    // 4th iteration - 0 missing keys (1 key del local, 1 key already deleted, 1 key missing but del in remote, 1 key
    // with ttl update missing but del remote): Remote returns: id0T, id1D, id3TD, idTD. Token index is pointing to idT.
    // 5th iteration - 0 missing keys (1 key - two records - missing but del remote, 2 puts already present but TTL
    // update of one of them is applied): Remote returns: idTD, b0T, b1T. b1 apply ttl update. Token index is pointing to
    // b1.
    // 6th iteration - 1 missing key (put + ttl update for a key, 1 deprecated id ignored, 1 TTL update already applied):
    // Remote returns: b1T, b2T, b3T, b0T. b2 missing, and ttl updated. b3 has no local key.
    // 7th iteration - 0 missing keys (2 TTL updates already applied, 1 TTL update of a deprecated ID ignored)
    // |1st iter |2nd iter|3rd iter|4th iter|5th iter|6th iter|7th iter|
    // L|id0|id1|id2|id0D|id1T|   |   |    |    |    |    |    |    |    |   |   |b0p|b1p|   |  |b0pT|    |    |    |id1D|id2T|id4|id4T|        |        |b1pT    |b2p|b2pT|
    // R|id0|id1|id2|    |    |id3|id4|id4T|id3T|id2T|id1T|id0T|id1D|id3D|idT|idD|b0 |b1 |b2 |b3|b0T |b1T |b2T | b3T|
    // E|id0|id1|id2|id0D|id1T|   |id4|id4T|    |id2T|    |    |id1D|    |   |   |b0p|b1p|b2p|  |b0pT|b1pT|b2pT|    |
    int[] missingKeysCounts = { 0, 1, 0, 0, 0, 1, 0 };
    int[] missingBuffersCount = { 5, 3, 3, 3, 2, 0, 0 };
    int expectedIndex = 0;
    int missingBuffersIndex = 0;
    for (int missingKeysCount : missingKeysCounts) {
        expectedIndex = Math.min(expectedIndex + batchSize, numMessagesInEachPart) - 1;
        List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos);
        assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
        for (int i = 0; i < response.size(); i++) {
            assertEquals(missingKeysCount, response.get(i).missingStoreMessages.size());
            assertEquals(expectedIndex, ((MockFindToken) response.get(i).remoteToken).getIndex());
            remoteReplicaInfos.get(i).setToken(response.get(i).remoteToken);
        }
        replicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos, response, false);
        for (int i = 0; i < response.size(); i++) {
            assertEquals("Token should have been set correctly in fixMissingStoreKeys()", response.get(i).remoteToken, remoteReplicaInfos.get(i).getToken());
        }
        missingBuffers = expectedLocalHost.getMissingBuffers(localHost.buffersByPartition);
        for (Map.Entry<PartitionId, List<ByteBuffer>> entry : missingBuffers.entrySet()) {
            assertEquals("Missing buffers count mismatch for iteration count " + missingBuffersIndex, missingBuffersCount[missingBuffersIndex], entry.getValue().size());
        }
        missingBuffersIndex++;
    }
    // no more missing keys
    List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
    for (ReplicaThread.ExchangeMetadataResponse metadata : response) {
        assertEquals(0, metadata.missingStoreMessages.size());
        assertEquals(expectedIndex, ((MockFindToken) metadata.remoteToken).getIndex());
    }
    missingBuffers = expectedLocalHost.getMissingBuffers(localHost.buffersByPartition);
    assertEquals("There should be no missing buffers", 0, missingBuffers.size());
    // validate everything
    for (Map.Entry<PartitionId, List<MessageInfo>> remoteInfoEntry : remoteHost.infosByPartition.entrySet()) {
        List<MessageInfo> remoteInfos = remoteInfoEntry.getValue();
        List<MessageInfo> localInfos = localHost.infosByPartition.get(remoteInfoEntry.getKey());
        Set<StoreKey> seen = new HashSet<>();
        for (MessageInfo remoteInfo : remoteInfos) {
            StoreKey remoteId = remoteInfo.getStoreKey();
            if (seen.add(remoteId)) {
                StoreKey localId = storeKeyConverter.convert(Collections.singleton(remoteId)).get(remoteId);
                MessageInfo localInfo = getMessageInfo(localId, localInfos, false, false, false);
                if (localId == null) {
                    // this is a deprecated ID. There should be no messages locally
                    assertNull(remoteId + " is deprecated and should have no entries", localInfo);
                } else {
                    MessageInfo mergedRemoteInfo = getMergedMessageInfo(remoteId, remoteInfos);
                    if (localInfo == null) {
                        // local has no put, must be deleted on remote
                        assertTrue(localId + ":" + remoteId + " not replicated", mergedRemoteInfo.isDeleted());
                    } else {
                        // local has a put and must be either at or beyond the state of the remote (based on ops above)
                        MessageInfo mergedLocalInfo = getMergedMessageInfo(localId, localInfos);
                        if (mergedRemoteInfo.isDeleted()) {
                            // delete on remote, should be deleted locally too
                            assertTrue(localId + ":" + remoteId + " is deleted on remote but not locally", mergedLocalInfo.isDeleted());
                        } else if (mergedRemoteInfo.isTtlUpdated() && !idsDeletedLocallyByPartition.get(remoteInfoEntry.getKey()).equals(localId)) {
                            // ttl updated on remote, should be ttl updated locally too
                            assertTrue(localId + ":" + remoteId + " is updated on remote but not locally", mergedLocalInfo.isTtlUpdated());
                        } else if (!idsDeletedLocallyByPartition.get(remoteInfoEntry.getKey()).equals(localId)) {
                            // should not be updated or deleted locally
                            assertFalse(localId + ":" + remoteId + " has been updated", mergedLocalInfo.isTtlUpdated());
                            assertFalse(localId + ":" + remoteId + " has been deleted", mergedLocalInfo.isDeleted());
                        }
                    }
                }
            }
        }
    }
}

Example 75 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class LeaderBasedReplicationTest method replicaThreadLeaderBasedReplicationFetchMissingKeysInMultipleCyclesTest.

/**
 * Test leader based replication to ensure token is advanced correctly for standby replicas when missing messages
 * are fetched via intra-dc replication in multiple cycles.
 * @throws Exception
 */
@Test
public void replicaThreadLeaderBasedReplicationFetchMissingKeysInMultipleCyclesTest() throws Exception {
    Pair<StorageManager, ReplicationManager> managers = createStorageManagerAndReplicationManager(clusterMap, clusterMapConfig, mockHelixParticipant);
    StorageManager storageManager = managers.getFirst();
    MockReplicationManager replicationManager = (MockReplicationManager) managers.getSecond();
    // set mock local stores on all remoteReplicaInfos which will used during replication.
    for (PartitionId partitionId : replicationManager.partitionToPartitionInfo.keySet()) {
        localHost.addStore(partitionId, null);
        Store localStore = localHost.getStore(partitionId);
        localStore.start();
        List<RemoteReplicaInfo> remoteReplicaInfos = replicationManager.partitionToPartitionInfo.get(partitionId).getRemoteReplicaInfos();
        remoteReplicaInfos.forEach(remoteReplicaInfo -> remoteReplicaInfo.setLocalStore(localStore));
    }
    // get remote replicas and replica thread for remote host on local datacenter
    ReplicaThread intraColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInLocalDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForLocalDC = intraColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInLocalDC);
    // get remote replicas and replica thread for remote host on remote datacenter
    ReplicaThread crossColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInRemoteDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForRemoteDC = crossColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInRemoteDC);
    // mock helix transition state from standby to leader for local leader partitions
    List<? extends ReplicaId> replicaIds = clusterMap.getReplicaIds(replicationManager.dataNodeId);
    for (ReplicaId replicaId : replicaIds) {
        MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
        if (mockReplicaId.getReplicaState() == ReplicaState.LEADER) {
            MockPartitionId mockPartitionId = (MockPartitionId) replicaId.getPartitionId();
            mockHelixParticipant.onPartitionBecomeLeaderFromStandby(mockPartitionId.toPathString());
        }
    }
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition = new HashMap<>();
    Map<PartitionId, List<StoreKey>> idsToBeTtlUpdatedByPartition = new HashMap<>();
    int numOfMessages = 5;
    for (PartitionId id : partitionIds) {
        List<StoreKey> toBeTtlUpdated = new ArrayList<>();
        // Adding 5 put messages to remote hosts
        List<StoreKey> ids = addPutMessagesToReplicasOfPartition(id, Arrays.asList(remoteHostInLocalDC, remoteHostInRemoteDC), numOfMessages);
        // update the TTL of first put message
        StoreKey toTtlUpdateId = ids.get(0);
        addTtlUpdateMessagesToReplicasOfPartition(id, toTtlUpdateId, Arrays.asList(remoteHostInLocalDC, remoteHostInRemoteDC), UPDATED_EXPIRY_TIME_MS);
        toBeTtlUpdated.add(toTtlUpdateId);
        idsToBeTtlUpdatedByPartition.put(id, toBeTtlUpdated);
    }
    // Choose partitions that are leaders on both local and remote nodes
    Set<ReplicaId> remoteLeaderReplicasWithLeaderPartitionsOnLocalNode = getRemoteLeaderReplicasWithLeaderPartitionsOnLocalNode(clusterMap, replicationManager.dataNodeId, remoteNodeInRemoteDC);
    // fetch all messages in the metadata exchange from remote replica on remote datacenter by setting batchSize
    // in the connection to 6
    // There are 6 records in the remote host. 5 put records + 1 ttl update record. We will receive 5 messages in the
    // metadata exchange as ttl update will be merged with its put record and sent as single record.
    List<ReplicaThread.ExchangeMetadataResponse> responseForRemoteNodeInRemoteDC = crossColoReplicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHostInRemoteDC, numOfMessages), remoteReplicaInfosForRemoteDC);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfosForRemoteDC.size(), responseForRemoteNodeInRemoteDC.size());
    for (ReplicaThread.ExchangeMetadataResponse exchangeMetadataResponse : responseForRemoteNodeInRemoteDC) {
        // we should have received 5 messages bo, b1, b2, b3, b4 (ttl_update for b0 will be merged with b0 put message)
        assertEquals("mismatch in number of messages received from remote replica", numOfMessages, exchangeMetadataResponse.missingStoreMessages.size());
    }
    // Filter leader replicas to fetch missing keys
    List<RemoteReplicaInfo> leaderReplicas = new ArrayList<>();
    List<ReplicaThread.ExchangeMetadataResponse> exchangeMetadataResponseListForLeaderReplicas = new ArrayList<>();
    crossColoReplicaThread.getLeaderReplicaList(remoteReplicaInfosForRemoteDC, responseForRemoteNodeInRemoteDC, leaderReplicas, exchangeMetadataResponseListForLeaderReplicas);
    // verify that only leader replicas in remoteHost2 are chosen for fetching missing messages.
    Set<ReplicaId> remoteReplicasToFetchInReplicaThread = leaderReplicas.stream().map(RemoteReplicaInfo::getReplicaId).collect(Collectors.toSet());
    assertThat("mismatch in leader remote replicas to fetch missing keys", remoteLeaderReplicasWithLeaderPartitionsOnLocalNode, is(remoteReplicasToFetchInReplicaThread));
    // fetch missing keys for leader replicas from remoteHost2
    if (leaderReplicas.size() > 0) {
        crossColoReplicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHostInRemoteDC, numOfMessages), leaderReplicas, exchangeMetadataResponseListForLeaderReplicas, false);
    }
    // missing messages are not fetched yet.
    for (int i = 0; i < remoteReplicaInfosForRemoteDC.size(); i++) {
        if (remoteLeaderReplicasWithLeaderPartitionsOnLocalNode.contains(remoteReplicaInfosForRemoteDC.get(i).getReplicaId())) {
            assertThat("remote Token should be updated for leader replica", remoteReplicaInfosForRemoteDC.get(i).getToken(), is(responseForRemoteNodeInRemoteDC.get(i).remoteToken));
        } else {
            assertThat("remote Token should not be updated for standby replica", remoteReplicaInfosForRemoteDC.get(i).getToken(), not(responseForRemoteNodeInRemoteDC.get(i).remoteToken));
        }
    }
    int numOfMessagesToBeFetchedFromRemoteHost1 = numOfMessages - 1;
    // Fetch only first 4 messages from remote host in local datacenter
    List<ReplicaThread.ExchangeMetadataResponse> responseForRemoteNodeInLocalDC = intraColoReplicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHostInLocalDC, numOfMessagesToBeFetchedFromRemoteHost1), remoteReplicaInfosForLocalDC);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfosForLocalDC.size(), responseForRemoteNodeInLocalDC.size());
    // fetch missing keys from remoteHost1
    intraColoReplicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHostInLocalDC, numOfMessagesToBeFetchedFromRemoteHost1), remoteReplicaInfosForLocalDC, responseForRemoteNodeInLocalDC, false);
    // verify that remote token has moved forward for all partitions since it is intra-dc replication.
    for (int i = 0; i < responseForRemoteNodeInLocalDC.size(); i++) {
        assertEquals("token mismatch for intra-dc replication", remoteReplicaInfosForLocalDC.get(i).getToken(), responseForRemoteNodeInLocalDC.get(i).remoteToken);
    }
    // process metadata response for cross-colo replicas. Missing keys b0, b1, b2 and b3 must have come now via intra-dc replication
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        crossColoReplicaThread.processMissingKeysFromPreviousMetadataResponse(remoteReplicaInfo);
    }
    // verify that the missingStoreMessages size in standby replicas is 1 as b0, b1, b2 and b3 are fetched and b4 is pending
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        if (!remoteLeaderReplicasWithLeaderPartitionsOnLocalNode.contains(remoteReplicaInfo.getReplicaId())) {
            assertEquals("mismatch in number of missing messages for remote standby replicas after intra-dc replication", remoteReplicaInfo.getExchangeMetadataResponse().missingStoreMessages.size(), 1);
        }
    }
    // Fetch remaining one message (b4) from remote host in local datacenter
    responseForRemoteNodeInLocalDC = intraColoReplicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHostInLocalDC, numOfMessagesToBeFetchedFromRemoteHost1), remoteReplicaInfosForLocalDC);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfosForLocalDC.size(), responseForRemoteNodeInLocalDC.size());
    // fetch missing keys from remoteHost1
    intraColoReplicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHostInLocalDC, numOfMessagesToBeFetchedFromRemoteHost1), remoteReplicaInfosForLocalDC, responseForRemoteNodeInLocalDC, false);
    // verify remote token for intra-dc replicas
    for (int i = 0; i < responseForRemoteNodeInLocalDC.size(); i++) {
        assertThat("mismatch in remote token after all missing messages are received", remoteReplicaInfosForLocalDC.get(i).getToken(), is(responseForRemoteNodeInLocalDC.get(i).remoteToken));
    }
    // process metadata response for cross-colo replicas. Missing keys b0, b1, b2 and b3 must have come now via intra-dc replication
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        crossColoReplicaThread.processMissingKeysFromPreviousMetadataResponse(remoteReplicaInfo);
    }
    // remote token for all replicas (leader and standby) should move forward.
    for (int i = 0; i < responseForRemoteNodeInRemoteDC.size(); i++) {
        assertThat("mismatch in remote token after all missing messages are received", remoteReplicaInfosForRemoteDC.get(i).getToken(), is(responseForRemoteNodeInRemoteDC.get(i).remoteToken));
    }
    // compare the final state of all messages and buffers are in sync at local node and remoteNodeInLocalDC
    checkBlobMessagesAreEqualInLocalAndRemoteHosts(localHost, remoteHostInLocalDC, idsToBeIgnoredByPartition, idsToBeTtlUpdatedByPartition);
    // compare the final state of all messages and buffers are in sync at local node and remoteNodeInRemoteDC
    checkBlobMessagesAreEqualInLocalAndRemoteHosts(localHost, remoteHostInRemoteDC, idsToBeIgnoredByPartition, idsToBeTtlUpdatedByPartition);
    storageManager.shutdown();
}