Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 96 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MockHost method getMissingInfos.

/**
 * Gets the message infos that are present in this host but missing in {@code other}.
 * @param other the list of {@link MessageInfo} to check against.
 * @return the message infos that are present in this host but missing in {@code other}.
 */
Map<PartitionId, List<MessageInfo>> getMissingInfos(Map<PartitionId, List<MessageInfo>> other, StoreKeyConverter storeKeyConverter) throws Exception {
    Map<PartitionId, List<MessageInfo>> missingInfos = new HashMap<>();
    for (Map.Entry<PartitionId, List<MessageInfo>> entry : infosByPartition.entrySet()) {
        PartitionId partitionId = entry.getKey();
        for (MessageInfo messageInfo : entry.getValue()) {
            boolean found = false;
            StoreKey convertedKey;
            if (storeKeyConverter == null) {
                convertedKey = messageInfo.getStoreKey();
            } else {
                Map<StoreKey, StoreKey> map = storeKeyConverter.convert(Collections.singletonList(messageInfo.getStoreKey()));
                convertedKey = map.get(messageInfo.getStoreKey());
                if (convertedKey == null) {
                    continue;
                }
            }
            for (MessageInfo otherInfo : other.get(partitionId)) {
                if (convertedKey.equals(otherInfo.getStoreKey()) && messageInfo.isDeleted() == otherInfo.isDeleted()) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                missingInfos.computeIfAbsent(partitionId, partitionId1 -> new ArrayList<>()).add(messageInfo);
            }
        }
    }
    return missingInfos;
}

Example 97 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTestHelper method checkBlobMessagesAreEqualInLocalAndRemoteHosts.

/**
 * Verifies that blob messages across all partitions at local and remote hosts are equal.
 */
protected void checkBlobMessagesAreEqualInLocalAndRemoteHosts(MockHost localHost, MockHost remoteHost, Map<PartitionId, List<StoreKey>> idsToBeIgnoredByPartition, Map<PartitionId, List<StoreKey>> idsToBeTtlUpdatedByPartition) {
    for (Map.Entry<PartitionId, List<MessageInfo>> remoteInfoEntry : remoteHost.infosByPartition.entrySet()) {
        List<MessageInfo> remoteInfos = remoteInfoEntry.getValue();
        List<MessageInfo> localInfos = localHost.infosByPartition.get(remoteInfoEntry.getKey());
        int remoteIndex = 0;
        Set<StoreKey> seen = new HashSet<>();
        for (MessageInfo remoteInfo : remoteInfos) {
            StoreKey id = remoteInfo.getStoreKey();
            if (seen.add(id)) {
                MessageInfo localInfo = getMessageInfo(id, localInfos, false, false, false);
                if (localInfo == null) {
                    assertTrue("Should be ignored", idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()).contains(id));
                } else {
                    assertFalse("Should not be ignored", idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()) != null && idsToBeIgnoredByPartition.get(remoteInfoEntry.getKey()).contains(id));
                    MessageInfo mergedLocalInfo = getMergedMessageInfo(id, localInfos);
                    MessageInfo mergedRemoteInfo = getMergedMessageInfo(id, remoteInfos);
                    assertEquals(mergedLocalInfo.isDeleted(), mergedRemoteInfo.isDeleted());
                    assertEquals(mergedLocalInfo.isTtlUpdated(), mergedRemoteInfo.isTtlUpdated());
                    assertEquals(mergedLocalInfo.isTtlUpdated(), idsToBeTtlUpdatedByPartition.get(remoteInfoEntry.getKey()).contains(id));
                    assertEquals(mergedLocalInfo.getLifeVersion(), mergedRemoteInfo.getLifeVersion());
                    assertEquals(mergedLocalInfo.getAccountId(), mergedRemoteInfo.getAccountId());
                    assertEquals(mergedLocalInfo.getContainerId(), mergedRemoteInfo.getContainerId());
                    assertEquals("Key " + id, mergedLocalInfo.getExpirationTimeInMs(), mergedRemoteInfo.getExpirationTimeInMs());
                    ByteBuffer putRecordBuffer = null;
                    for (int i = 0; i < localInfos.size(); i++) {
                        if (localInfo.equals(localInfos.get(i))) {
                            putRecordBuffer = localHost.buffersByPartition.get(remoteInfoEntry.getKey()).get(i).duplicate();
                            break;
                        }
                    }
                    assertNotNull(putRecordBuffer);
                    // Make sure the put buffer contains the same info as the message Info
                    assertPutRecord(putRecordBuffer, remoteHost.buffersByPartition.get(remoteInfoEntry.getKey()).get(remoteIndex).duplicate(), mergedLocalInfo);
                }
            }
            remoteIndex++;
        }
    }
}

Example 98 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.

public static void addPutMessagesToReplicasOfPartition(List<StoreKey> ids, List<Transformer> transformPerId, List<MockHost> hosts) throws MessageFormatException, IOException {
    Iterator<Transformer> transformerIterator = transformPerId.iterator();
    for (StoreKey storeKey : ids) {
        Transformer transformer = transformerIterator.next();
        BlobId id = (BlobId) storeKey;
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, id.getAccountId(), id.getContainerId(), BlobId.isEncrypted(id.toString()));
        MessageInfo msgInfo = msgInfoAndBuffer.messageInfo;
        ByteBuffer byteBuffer = msgInfoAndBuffer.byteBuffer;
        if (transformer != null) {
            Message message = new Message(msgInfo, new ByteBufferInputStream(byteBuffer));
            TransformationOutput output = transformer.transform(message);
            assertNull(output.getException());
            message = output.getMsg();
            byteBuffer = ByteBuffer.wrap(Utils.readBytesFromStream(message.getStream(), (int) message.getMessageInfo().getSize()));
            msgInfo = message.getMessageInfo();
        }
        for (MockHost host : hosts) {
            host.addMessage(id.getPartition(), msgInfo, byteBuffer.duplicate());
        }
    }
}

Example 99 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.

/**
 * For the given partitionId, constructs a PUT messages and adds it to the given hosts.
 * @param id the {@link StoreKey} to create a put message for.
 * @param accountId the accountId of this message.
 * @param containerId the containerId of this message.
 * @param partitionId the {@link PartitionId} to use for generating the {@link StoreKey} of the message.
 * @param hosts the list of {@link MockHost} all of which will be populated with the messages.
 * @param operationTime the operation of this message.
 * @param expirationTime the expirationTime of this message.
 * @throws MessageFormatException
 * @throws IOException
 */
public static void addPutMessagesToReplicasOfPartition(StoreKey id, short accountId, short containerId, PartitionId partitionId, List<MockHost> hosts, long operationTime, long expirationTime) throws MessageFormatException, IOException {
    // add a PUT message with expiration time less than VCR threshold to remote host.
    boolean toEncrypt = TestUtils.RANDOM.nextBoolean();
    ReplicationTest.PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, accountId, containerId, toEncrypt);
    for (MockHost host : hosts) {
        host.addMessage(partitionId, new MessageInfo(id, msgInfoAndBuffer.byteBuffer.remaining(), false, false, false, expirationTime, null, accountId, containerId, operationTime, (short) 0), msgInfoAndBuffer.byteBuffer);
    }
}

Example 100 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTestHelper method lifeVersionLocalGreaterThanRemote_Delete.

/**
 * Helepr function to test when the local lifeVersion is greater than the remote lifeVersion.
 * @param localTtlUpdated
 * @param remoteTtlUpdated
 * @throws Exception
 */
protected void lifeVersionLocalGreaterThanRemote_Delete(boolean localTtlUpdated, boolean remoteTtlUpdated) throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        // add 1 messages to remote host with lifeVersion being 0 and add it local host with lifeVersion being 1.
        StoreKey toDeleteId = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost), 1).get(0);
        if (remoteTtlUpdated) {
            addTtlUpdateMessagesToReplicasOfPartition(partitionId, toDeleteId, Arrays.asList(remoteHost), UPDATED_EXPIRY_TIME_MS);
        }
        addDeleteMessagesToReplicasOfPartition(partitionId, toDeleteId, Collections.singletonList(remoteHost));
        BlobId blobId = (BlobId) toDeleteId;
        short accountId = blobId.getAccountId();
        short containerId = blobId.getContainerId();
        short lifeVersion = 1;
        // first put message has encryption turned on
        boolean toEncrypt = true;
        // create a put message with lifeVersion bigger than 0
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(toDeleteId, accountId, containerId, toEncrypt, lifeVersion);
        localHost.addMessage(partitionId, new MessageInfo(toDeleteId, msgInfoAndBuffer.byteBuffer.remaining(), false, false, false, Utils.Infinite_Time, null, accountId, containerId, msgInfoAndBuffer.messageInfo.getOperationTimeMs(), lifeVersion), msgInfoAndBuffer.byteBuffer);
        if (localTtlUpdated) {
            addTtlUpdateMessagesToReplicasOfPartition(partitionId, toDeleteId, Collections.singletonList(localHost), EXPIRY_TIME_MS, lifeVersion);
        }
        // ensure that the first key is not deleted in the local host
        assertNull(toDeleteId + " should not be deleted in the local host", getMessageInfo(toDeleteId, localHost.infosByPartition.get(partitionId), true, false, false));
    }
    int batchSize = 4;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, null, null, null);
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasAndThread.getFirst().get(remoteHost.dataNodeId);
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    // Do the replica metadata exchange.
    List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
    for (int i = 0; i < response.size(); i++) {
        // we don't have any missing key here.
        assertEquals(0, response.get(i).missingStoreMessages.size());
        remoteReplicaInfos.get(i).setToken(response.get(i).remoteToken);
        PartitionId partitionId = partitionIds.get(i);
        StoreKey key = localHost.infosByPartition.get(partitionId).get(0).getStoreKey();
        assertNull(key + " should not be deleted in the local host", getMessageInfo(key, localHost.infosByPartition.get(partitionId), true, false, false));
        if (!localTtlUpdated) {
            assertNull(key + " should not be ttlUpdated in the local host", getMessageInfo(key, localHost.infosByPartition.get(partitionId), false, false, true));
        }
    }
}