Examples with StoreKey com.github.ambry.store.StoreKey used on opensource projects

Example 76 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class LeaderBasedReplicationTest method replicaThreadLeaderBasedReplicationForTTLUpdatesDeleteAndUndeleteMessagesTest.

/**
 * Test leader based replication to ensure token is advanced correctly and blob properties ttl
 * _update, delete, undelete are applied correctly when missing messages in standby replicas
 * are fetched via intra-dc replication.
 * @throws Exception
 */
@Test
public void replicaThreadLeaderBasedReplicationForTTLUpdatesDeleteAndUndeleteMessagesTest() throws Exception {
    Pair<StorageManager, ReplicationManager> managers = createStorageManagerAndReplicationManager(clusterMap, clusterMapConfig, mockHelixParticipant);
    StorageManager storageManager = managers.getFirst();
    MockReplicationManager replicationManager = (MockReplicationManager) managers.getSecond();
    int batchSize = 10;
    // set mock local stores on all remoteReplicaInfos which will used during replication.
    for (PartitionId partitionId : replicationManager.partitionToPartitionInfo.keySet()) {
        localHost.addStore(partitionId, null);
        Store localStore = localHost.getStore(partitionId);
        localStore.start();
        List<RemoteReplicaInfo> remoteReplicaInfos = replicationManager.partitionToPartitionInfo.get(partitionId).getRemoteReplicaInfos();
        remoteReplicaInfos.forEach(remoteReplicaInfo -> remoteReplicaInfo.setLocalStore(localStore));
    }
    // get remote replicas and replica thread for remote host on local datacenter
    ReplicaThread intraColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInLocalDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForLocalDC = intraColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInLocalDC);
    // get remote replicas and replica thread for remote host on remote datacenter
    ReplicaThread crossColoReplicaThread = replicationManager.dataNodeIdToReplicaThread.get(remoteNodeInRemoteDC);
    List<RemoteReplicaInfo> remoteReplicaInfosForRemoteDC = crossColoReplicaThread.getRemoteReplicaInfos().get(remoteNodeInRemoteDC);
    // mock helix transition state from standby to leader for local leader partitions
    List<? extends ReplicaId> replicaIds = clusterMap.getReplicaIds(replicationManager.dataNodeId);
    for (ReplicaId replicaId : replicaIds) {
        MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
        if (mockReplicaId.getReplicaState() == ReplicaState.LEADER) {
            MockPartitionId mockPartitionId = (MockPartitionId) replicaId.getPartitionId();
            mockHelixParticipant.onPartitionBecomeLeaderFromStandby(mockPartitionId.toPathString());
        }
    }
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition = new HashMap<>();
    Map<PartitionId, List<StoreKey>> idsToBeTtlUpdatedByPartition = new HashMap<>();
    for (PartitionId id : partitionIds) {
        List<StoreKey> toBeIgnored = new ArrayList<>();
        List<StoreKey> toBeUndeleted = new ArrayList<>();
        // Adding 4 PUT messages b0, b1, b2, b3 to remoteNodeInLocalDC and remoteNodeInRemoteDC
        List<StoreKey> ids = addPutMessagesToReplicasOfPartition(id, Arrays.asList(remoteHostInLocalDC, remoteHostInRemoteDC), 4);
        // ttl update to be added to b1, b2, b3
        List<StoreKey> toBeTtlUpdated = new ArrayList<>(ids);
        toBeTtlUpdated.remove(ids.get(0));
        // delete to be added to b2, b3
        toBeIgnored.add(ids.get(2));
        toBeIgnored.add(ids.get(3));
        // un-delete to be added to b3
        toBeUndeleted.add(ids.get(3));
        toBeIgnored.remove(ids.get(3));
        // Add TTLUpdate records for blobs b1,b2,b3 in remoteNodeInLocalDC and remoteNodeInRemoteDC
        for (int j = 0; j < toBeTtlUpdated.size(); j++) {
            addTtlUpdateMessagesToReplicasOfPartition(id, toBeTtlUpdated.get(j), Collections.singletonList(remoteHostInLocalDC), UPDATED_EXPIRY_TIME_MS);
            addTtlUpdateMessagesToReplicasOfPartition(id, toBeTtlUpdated.get(toBeTtlUpdated.size() - 1 - j), Collections.singletonList(remoteHostInRemoteDC), UPDATED_EXPIRY_TIME_MS);
        }
        // Add delete records for blobs b2,b3 in remoteNodeInLocalDC and remoteNodeInRemoteDC
        for (int j = 0; j < toBeIgnored.size(); j++) {
            addDeleteMessagesToReplicasOfPartition(id, toBeIgnored.get(j), Collections.singletonList(remoteHostInLocalDC), (short) 0, EXPIRY_TIME_MS);
            addDeleteMessagesToReplicasOfPartition(id, toBeIgnored.get(toBeIgnored.size() - 1 - j), Collections.singletonList(remoteHostInRemoteDC), (short) 0, EXPIRY_TIME_MS);
        }
        // Add un-delete records for blob b3 with life_version as 1 in remoteNodeInLocalDC and remoteNodeInRemoteDC
        for (StoreKey storeKey : toBeUndeleted) {
            addUndeleteMessagesToReplicasOfPartition(id, storeKey, Arrays.asList(remoteHostInLocalDC, remoteHostInRemoteDC), (short) 1);
        }
        // will be used later while comparing the final message records in local and remote nodes
        idsToBeIgnoredByPartition.put(id, toBeIgnored);
        idsToBeTtlUpdatedByPartition.put(id, toBeTtlUpdated);
    }
    // Inter-dc replication
    // Send metadata request to remoteNodeInRemoteDC to fetch missing keys information.
    List<ReplicaThread.ExchangeMetadataResponse> responseForRemoteNodeInRemoteDC = crossColoReplicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHostInRemoteDC, batchSize), remoteReplicaInfosForRemoteDC);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfosForRemoteDC.size(), responseForRemoteNodeInRemoteDC.size());
    // Filter leader replicas to fetch missing keys
    List<RemoteReplicaInfo> leaderReplicas = new ArrayList<>();
    List<ReplicaThread.ExchangeMetadataResponse> exchangeMetadataResponseListForLeaderReplicas = new ArrayList<>();
    crossColoReplicaThread.getLeaderReplicaList(remoteReplicaInfosForRemoteDC, responseForRemoteNodeInRemoteDC, leaderReplicas, exchangeMetadataResponseListForLeaderReplicas);
    // verify that only leader partitions in local and remote nodes are chosen for fetching missing messages.
    Set<ReplicaId> remoteLeaderReplicasWithLeaderPartitionsOnLocalNode = getRemoteLeaderReplicasWithLeaderPartitionsOnLocalNode(clusterMap, replicationManager.dataNodeId, remoteNodeInRemoteDC);
    Set<ReplicaId> leaderReplicaSetInReplicaThread = leaderReplicas.stream().map(RemoteReplicaInfo::getReplicaId).collect(Collectors.toSet());
    assertThat("mismatch in leader remote replicas to fetch missing keys", remoteLeaderReplicasWithLeaderPartitionsOnLocalNode, is(leaderReplicaSetInReplicaThread));
    // fetch missing keys for leader replicas from remoteNodeInRemoteDC
    if (leaderReplicas.size() > 0) {
        crossColoReplicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHostInRemoteDC, batchSize), leaderReplicas, exchangeMetadataResponseListForLeaderReplicas, false);
    }
    // For standby replicas, token index will remain 0 and metadata information would be stored.
    for (int i = 0; i < remoteReplicaInfosForRemoteDC.size(); i++) {
        if (remoteLeaderReplicasWithLeaderPartitionsOnLocalNode.contains(remoteReplicaInfosForRemoteDC.get(i).getReplicaId())) {
            assertEquals("remote Token should be updated for leader replica", remoteReplicaInfosForRemoteDC.get(i).getToken(), (responseForRemoteNodeInRemoteDC.get(i).remoteToken));
        } else {
            assertThat("remote Token should not be updated for standby replica", remoteReplicaInfosForRemoteDC.get(i).getToken(), not(responseForRemoteNodeInRemoteDC.get(i).remoteToken));
            assertEquals("missing messages in metadata exchange should be stored for standby replica", remoteReplicaInfosForRemoteDC.get(i).getExchangeMetadataResponse().missingStoreMessages.size(), responseForRemoteNodeInRemoteDC.get(i).missingStoreMessages.size());
        }
    }
    // Intra-dc replication
    List<ReplicaThread.ExchangeMetadataResponse> responseForRemoteNodeInLocalDC = intraColoReplicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHostInLocalDC, batchSize), remoteReplicaInfosForLocalDC);
    intraColoReplicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(remoteHostInLocalDC, batchSize), remoteReplicaInfosForLocalDC, responseForRemoteNodeInLocalDC, false);
    // Verify that the remote token for all intra-colo replicas has been moved
    for (int i = 0; i < responseForRemoteNodeInLocalDC.size(); i++) {
        assertEquals(remoteReplicaInfosForLocalDC.get(i).getToken(), responseForRemoteNodeInLocalDC.get(i).remoteToken);
    }
    // arrived via intra-dc replication
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfosForRemoteDC) {
        crossColoReplicaThread.processMissingKeysFromPreviousMetadataResponse(remoteReplicaInfo);
    }
    // arrived via intra-dc replication
    for (int i = 0; i < responseForRemoteNodeInLocalDC.size(); i++) {
        assertEquals(remoteReplicaInfosForRemoteDC.get(i).getToken(), responseForRemoteNodeInRemoteDC.get(i).remoteToken);
    }
    // compare the messages and buffers are in sync at local host and remote host1
    checkBlobMessagesAreEqualInLocalAndRemoteHosts(localHost, remoteHostInLocalDC, idsToBeIgnoredByPartition, idsToBeTtlUpdatedByPartition);
    // compare the messages and buffers are in sync at local host and remote host1
    checkBlobMessagesAreEqualInLocalAndRemoteHosts(localHost, remoteHostInRemoteDC, idsToBeIgnoredByPartition, idsToBeTtlUpdatedByPartition);
    storageManager.shutdown();
}

Example 77 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MockHost method getMissingInfos.

/**
 * Gets the message infos that are present in this host but missing in {@code other}.
 * @param other the list of {@link MessageInfo} to check against.
 * @return the message infos that are present in this host but missing in {@code other}.
 */
Map<PartitionId, List<MessageInfo>> getMissingInfos(Map<PartitionId, List<MessageInfo>> other, StoreKeyConverter storeKeyConverter) throws Exception {
    Map<PartitionId, List<MessageInfo>> missingInfos = new HashMap<>();
    for (Map.Entry<PartitionId, List<MessageInfo>> entry : infosByPartition.entrySet()) {
        PartitionId partitionId = entry.getKey();
        for (MessageInfo messageInfo : entry.getValue()) {
            boolean found = false;
            StoreKey convertedKey;
            if (storeKeyConverter == null) {
                convertedKey = messageInfo.getStoreKey();
            } else {
                Map<StoreKey, StoreKey> map = storeKeyConverter.convert(Collections.singletonList(messageInfo.getStoreKey()));
                convertedKey = map.get(messageInfo.getStoreKey());
                if (convertedKey == null) {
                    continue;
                }
            }
            for (MessageInfo otherInfo : other.get(partitionId)) {
                if (convertedKey.equals(otherInfo.getStoreKey()) && messageInfo.isDeleted() == otherInfo.isDeleted()) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                missingInfos.computeIfAbsent(partitionId, partitionId1 -> new ArrayList<>()).add(messageInfo);
            }
        }
    }
    return missingInfos;
}

Example 78 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTestHelper method createMixedMessagesOnRemoteHost.

/**
 * Set up different combinations of PUT, DELETE messages on remote host.
 * For simplicity, we use O(old) to denote Version_2 blobId, N(new) to denote Version_5 blobId.
 * For example, OP means Version_2 PUT message, ND means Version_5 DELETE message, etc.
 * @param testSetup the {@link ReplicationTestSetup} used to provide test environment info.
 * @param msgStr the string presenting the sequence of PUT, DELETE messages
 * @throws Exception
 */
protected void createMixedMessagesOnRemoteHost(ReplicationTestSetup testSetup, String msgStr) throws Exception {
    PartitionId partitionId = testSetup.partitionIds.get(0);
    StoreKey oldKey = testSetup.oldKey;
    StoreKey newKey = testSetup.newKey;
    MockHost remoteHost = testSetup.remoteHost;
    String[] messages = msgStr.split("\\s");
    for (String message : messages) {
        switch(message) {
            case "OP":
                addPutMessagesToReplicasOfPartition(Collections.singletonList(oldKey), Collections.singletonList(remoteHost));
                break;
            case "OD":
                addDeleteMessagesToReplicasOfPartition(partitionId, oldKey, Collections.singletonList(remoteHost));
                break;
            case "NP":
                addPutMessagesToReplicasOfPartition(Collections.singletonList(newKey), Collections.singletonList(remoteHost));
                break;
            case "ND":
                addDeleteMessagesToReplicasOfPartition(partitionId, newKey, Collections.singletonList(remoteHost));
                break;
        }
    }
}

Example 79 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTestHelper method checkBlobMessagesAreEqualInLocalAndRemoteHosts.

/**
 * Verifies that blob messages across all partitions at local and remote hosts are equal.
 */
protected void checkBlobMessagesAreEqualInLocalAndRemoteHosts(MockHost localHost, MockHost remoteHost, Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition, Map<PartitionId, List<StoreKey>> idsToBeTtlUpdatedByPartition) {
    for (Map.Entry<PartitionId, List<MessageInfo>> remoteInfoEntry : remoteHost.infosByPartition.entrySet()) {
        List<MessageInfo> remoteInfos = remoteInfoEntry.getValue();
        List<MessageInfo> localInfos = localHost.infosByPartition.get(remoteInfoEntry.getKey());
        int remoteIndex = 0;
        Set<StoreKey> seen = new HashSet<>();
        for (MessageInfo remoteInfo : remoteInfos) {
            StoreKey id = remoteInfo.getStoreKey();
            if (seen.add(id)) {
                MessageInfo localInfo = getMessageInfo(id, localInfos, false, false, false);
                if (localInfo == null) {
                    assertTrue("Should be ignored", idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()).contains(id));
                } else {
                    assertFalse("Should not be ignored", idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()) != null && idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()).contains(id));
                    MessageInfo mergedLocalInfo = getMergedMessageInfo(id, localInfos);
                    MessageInfo mergedRemoteInfo = getMergedMessageInfo(id, remoteInfos);
                    assertEquals(mergedLocalInfo.isDeleted(), mergedRemoteInfo.isDeleted());
                    assertEquals(mergedLocalInfo.isTtlUpdated(), mergedRemoteInfo.isTtlUpdated());
                    assertEquals(mergedLocalInfo.isTtlUpdated(), idsToBeTtlUpdatedByPartition.get(remoteInfoEntry.getKey()).contains(id));
                    assertEquals(mergedLocalInfo.getLifeVersion(), mergedRemoteInfo.getLifeVersion());
                    assertEquals(mergedLocalInfo.getAccountId(), mergedRemoteInfo.getAccountId());
                    assertEquals(mergedLocalInfo.getContainerId(), mergedRemoteInfo.getContainerId());
                    assertEquals("Key " + id, mergedLocalInfo.getExpirationTimeInMs(), mergedRemoteInfo.getExpirationTimeInMs());
                    ByteBuffer putRecordBuffer = null;
                    for (int i = 0; i < localInfos.size(); i++) {
                        if (localInfo.equals(localInfos.get(i))) {
                            putRecordBuffer = localHost.buffersByPartition.get(remoteInfoEntry.getKey()).get(i).duplicate();
                            break;
                        }
                    }
                    assertNotNull(putRecordBuffer);
                    // Make sure the put buffer contains the same info as the message Info
                    assertPutRecord(putRecordBuffer, remoteHost.buffersByPartition.get(remoteInfoEntry.getKey()).get(remoteIndex).duplicate(), mergedLocalInfo);
                }
            }
            remoteIndex++;
        }
    }
}

Example 80 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.

public static void addPutMessagesToReplicasOfPartition(List<StoreKey> ids, List<Transformer> transformPerId, List<MockHost> hosts) throws MessageFormatException, IOException {
    Iterator<Transformer> transformerIterator = transformPerId.iterator();
    for (StoreKey storeKey : ids) {
        Transformer transformer = transformerIterator.next();
        BlobId id = (BlobId) storeKey;
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, id.getAccountId(), id.getContainerId(), BlobId.isEncrypted(id.toString()));
        MessageInfo msgInfo = msgInfoAndBuffer.messageInfo;
        ByteBuffer byteBuffer = msgInfoAndBuffer.byteBuffer;
        if (transformer != null) {
            Message message = new Message(msgInfo, new ByteBufferInputStream(byteBuffer));
            TransformationOutput output = transformer.transform(message);
            assertNull(output.getException());
            message = output.getMsg();
            byteBuffer = ByteBuffer.wrap(Utils.readBytesFromStream(message.getStream(), (int) message.getMessageInfo().getSize()));
            msgInfo = message.getMessageInfo();
        }
        for (MockHost host : hosts) {
            host.addMessage(id.getPartition(), msgInfo, byteBuffer.duplicate());
        }
    }
}