Examples with Message com.github.ambry.store.Message used on opensource projects

Example 6 with Message

use of com.github.ambry.store.Message in project ambry by linkedin.

the class ValidatingTransformer method transform.

@Override
public TransformationOutput transform(Message message) {
    ByteBuffer encryptionKey;
    BlobProperties props;
    ByteBuffer metadata;
    BlobData blobData;
    MessageInfo msgInfo = message.getMessageInfo();
    InputStream msgStream = message.getStream();
    TransformationOutput transformationOutput = null;
    try {
        // Read header
        ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
        msgStream.read(headerVersion.array());
        short version = headerVersion.getShort();
        if (!isValidHeaderVersion(version)) {
            throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
        }
        int headerSize = getHeaderSizeForVersion(version);
        ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize);
        headerBuffer.put(headerVersion.array());
        msgStream.read(headerBuffer.array(), Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
        headerBuffer.rewind();
        MessageHeader_Format header = getMessageHeader(version, headerBuffer);
        header.verifyHeader();
        StoreKey keyInStream = storeKeyFactory.getStoreKey(new DataInputStream(msgStream));
        if (header.isPutRecord()) {
            if (header.hasLifeVersion() && header.getLifeVersion() != msgInfo.getLifeVersion()) {
                logger.trace("LifeVersion in stream: {} failed to match lifeVersion from Index: {} for key {}", header.getLifeVersion(), msgInfo.getLifeVersion(), keyInStream);
            }
            encryptionKey = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(msgStream) : null;
            props = deserializeBlobProperties(msgStream);
            metadata = deserializeUserMetadata(msgStream);
            blobData = deserializeBlob(msgStream);
        } else {
            throw new IllegalStateException("Message cannot be anything rather than put record ");
        }
        if (msgInfo.getStoreKey().equals(keyInStream)) {
            // BlobIDTransformer only exists on ambry-server and replication between servers is relying on blocking channel
            // which is still using java ByteBuffer. So, no need to consider releasing stuff.
            // @todo, when netty Bytebuf is adopted for blocking channel on ambry-server, remember to release this ByteBuf.
            PutMessageFormatInputStream transformedStream = new PutMessageFormatInputStream(keyInStream, encryptionKey, props, metadata, new ByteBufInputStream(blobData.content(), true), blobData.getSize(), blobData.getBlobType(), msgInfo.getLifeVersion());
            MessageInfo transformedMsgInfo = new MessageInfo.Builder(msgInfo).size(transformedStream.getSize()).isDeleted(false).isUndeleted(false).build();
            transformationOutput = new TransformationOutput(new Message(transformedMsgInfo, transformedStream));
        } else {
            throw new IllegalStateException("StoreKey in stream: " + keyInStream + " failed to match store key from Index: " + msgInfo.getStoreKey());
        }
    } catch (Exception e) {
        transformationOutput = new TransformationOutput(e);
    }
    return transformationOutput;
}

Example 7 with Message

use of com.github.ambry.store.Message in project ambry by linkedin.

the class ReplicationTestHelper method replicateAndVerify.

/**
 * Replicate between local and remote hosts and verify the results on local host are expected.
 * 1.remote host has different versions of blobIds and local host is empty;
 * 2.remote and local hosts have different conversion maps (StoreKeyConverter).
 * @param testSetup the {@link ReplicationTestSetup} used to provide test environment info.
 * @param expectedStr the string presenting expected sequence of PUT, DELETE messages on local host.
 * @throws Exception
 */
protected void replicateAndVerify(ReplicationTestSetup testSetup, String expectedStr) throws Exception {
    PartitionId partitionId = testSetup.partitionIds.get(0);
    List<RemoteReplicaInfo> singleReplicaList = testSetup.replicasToReplicate.get(testSetup.remoteHost.dataNodeId).stream().filter(e -> e.getReplicaId().getPartitionId() == partitionId).collect(Collectors.toList());
    // Do the replica metadata exchange.
    List<ReplicaThread.ExchangeMetadataResponse> responses = testSetup.replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(testSetup.remoteHost, 10, testSetup.remoteConversionMap), singleReplicaList);
    // Do Get request to fix missing keys
    testSetup.replicaThread.fixMissingStoreKeys(new MockConnectionPool.MockConnection(testSetup.remoteHost, 10, testSetup.remoteConversionMap), singleReplicaList, responses, false);
    // Verify
    String[] expectedResults = expectedStr.equals("") ? new String[0] : expectedStr.split("\\s");
    int size = testSetup.localHost.infosByPartition.get(partitionId).size();
    assertEquals("Mismatch in number of messages on local host after replication", expectedResults.length, size);
    for (int i = 0; i < size; ++i) {
        String blobIdStr = testSetup.localHost.infosByPartition.get(partitionId).get(i).getStoreKey().toString();
        boolean isDeleteMessage = testSetup.localHost.infosByPartition.get(partitionId).get(i).isDeleted();
        switch(expectedResults[i]) {
            case "OP":
                assertEquals("Mismatch in blodId on local host after replication", testSetup.oldKey.toString(), blobIdStr);
                assertFalse("Mismatch in message type on local host after replication", isDeleteMessage);
                break;
            case "OD":
                assertEquals("Mismatch in blodId on local host after replication", testSetup.oldKey.toString(), blobIdStr);
                assertTrue("Mismatch in message type on local host after replication", isDeleteMessage);
                break;
            case "NP":
                assertEquals("Mismatch in blodId on local host after replication", testSetup.newKey.toString(), blobIdStr);
                assertFalse("Mismatch in message type on local host after replication", isDeleteMessage);
                break;
            case "ND":
                assertEquals("Mismatch in blodId on local host after replication", testSetup.newKey.toString(), blobIdStr);
                assertTrue("Mismatch in message type on local host after replication", isDeleteMessage);
                break;
        }
    }
}

Example 8 with Message

use of com.github.ambry.store.Message in project ambry by linkedin.

the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.

public static void addPutMessagesToReplicasOfPartition(List<StoreKey> ids, List<Transformer> transformPerId, List<MockHost> hosts) throws MessageFormatException, IOException {
    Iterator<Transformer> transformerIterator = transformPerId.iterator();
    for (StoreKey storeKey : ids) {
        Transformer transformer = transformerIterator.next();
        BlobId id = (BlobId) storeKey;
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, id.getAccountId(), id.getContainerId(), BlobId.isEncrypted(id.toString()));
        MessageInfo msgInfo = msgInfoAndBuffer.messageInfo;
        ByteBuffer byteBuffer = msgInfoAndBuffer.byteBuffer;
        if (transformer != null) {
            Message message = new Message(msgInfo, new ByteBufferInputStream(byteBuffer));
            TransformationOutput output = transformer.transform(message);
            assertNull(output.getException());
            message = output.getMsg();
            byteBuffer = ByteBuffer.wrap(Utils.readBytesFromStream(message.getStream(), (int) message.getMessageInfo().getSize()));
            msgInfo = message.getMessageInfo();
        }
        for (MockHost host : hosts) {
            host.addMessage(id.getPartition(), msgInfo, byteBuffer.duplicate());
        }
    }
}