Examples with TransformationOutput com.github.ambry.store.TransformationOutput used on opensource projects

Example 6 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformer method transform.

@Override
public TransformationOutput transform(Message message) {
    Message transformedMsg = null;
    try {
        Objects.requireNonNull(message, "message must not be null");
        Objects.requireNonNull(message.getMessageInfo(), "message's messageInfo must not be null");
        Objects.requireNonNull(message.getStream(), "message's inputStream must not be null");
        StoreKey oldStoreKey = message.getMessageInfo().getStoreKey();
        StoreKey newStoreKey = storeKeyConverter.getConverted(oldStoreKey);
        if (newStoreKey != null) {
            transformedMsg = newMessage(message.getStream(), newStoreKey, message.getMessageInfo());
        }
    } catch (Exception e) {
        return new TransformationOutput(e);
    }
    return new TransformationOutput(transformedMsg);
}

Example 7 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class ValidatingKeyConvertingTransformer method transform.

@Override
public TransformationOutput transform(Message message) {
    ByteBuffer encryptionKey;
    BlobProperties props;
    ByteBuffer metadata;
    BlobData blobData;
    MessageInfo msgInfo = message.getMessageInfo();
    InputStream msgStream = message.getStream();
    TransformationOutput transformationOutput;
    try {
        // Read header
        ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
        msgStream.read(headerVersion.array());
        short version = headerVersion.getShort();
        if (!isValidHeaderVersion(version)) {
            throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
        }
        int headerSize = getHeaderSizeForVersion(version);
        ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize);
        headerBuffer.put(headerVersion.array());
        msgStream.read(headerBuffer.array(), Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
        headerBuffer.rewind();
        MessageHeader_Format header = getMessageHeader(version, headerBuffer);
        header.verifyHeader();
        StoreKey originalKey = storeKeyFactory.getStoreKey(new DataInputStream(msgStream));
        if (header.isPutRecord()) {
            encryptionKey = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(msgStream) : null;
            props = deserializeBlobProperties(msgStream);
            metadata = deserializeUserMetadata(msgStream);
            blobData = deserializeBlob(msgStream);
        } else {
            throw new IllegalArgumentException("Message cannot be a deleted record ");
        }
        if (msgInfo.getStoreKey().equals(originalKey)) {
            StoreKey newKey = storeKeyConverter.convert(Collections.singletonList(originalKey)).get(originalKey);
            if (newKey == null) {
                System.out.println("No mapping for the given key, transformed message will be null");
                transformationOutput = new TransformationOutput((Message) null);
            } else {
                MessageInfo transformedMsgInfo;
                PutMessageFormatInputStream transformedStream = new PutMessageFormatInputStream(newKey, encryptionKey, props, metadata, new ByteBufInputStream(blobData.content(), true), blobData.getSize(), blobData.getBlobType(), msgInfo.getLifeVersion());
                transformedMsgInfo = new MessageInfo.Builder(msgInfo).storeKey(newKey).size(transformedStream.getSize()).isUndeleted(false).build();
                transformationOutput = new TransformationOutput(new Message(transformedMsgInfo, transformedStream));
            }
        } else {
            throw new IllegalStateException("StoreKey in log " + originalKey + " failed to match store key from Index " + msgInfo.getStoreKey());
        }
    } catch (Exception e) {
        transformationOutput = new TransformationOutput(e);
    }
    return transformationOutput;
}

Example 8 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class ValidatingTransformer method transform.

@Override
public TransformationOutput transform(Message message) {
    ByteBuffer encryptionKey;
    BlobProperties props;
    ByteBuffer metadata;
    BlobData blobData;
    MessageInfo msgInfo = message.getMessageInfo();
    InputStream msgStream = message.getStream();
    TransformationOutput transformationOutput = null;
    try {
        // Read header
        ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
        msgStream.read(headerVersion.array());
        short version = headerVersion.getShort();
        if (!isValidHeaderVersion(version)) {
            throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
        }
        int headerSize = getHeaderSizeForVersion(version);
        ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize);
        headerBuffer.put(headerVersion.array());
        msgStream.read(headerBuffer.array(), Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
        headerBuffer.rewind();
        MessageHeader_Format header = getMessageHeader(version, headerBuffer);
        header.verifyHeader();
        StoreKey keyInStream = storeKeyFactory.getStoreKey(new DataInputStream(msgStream));
        if (header.isPutRecord()) {
            if (header.hasLifeVersion() && header.getLifeVersion() != msgInfo.getLifeVersion()) {
                logger.trace("LifeVersion in stream: {} failed to match lifeVersion from Index: {} for key {}", header.getLifeVersion(), msgInfo.getLifeVersion(), keyInStream);
            }
            encryptionKey = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(msgStream) : null;
            props = deserializeBlobProperties(msgStream);
            metadata = deserializeUserMetadata(msgStream);
            blobData = deserializeBlob(msgStream);
        } else {
            throw new IllegalStateException("Message cannot be anything rather than put record ");
        }
        if (msgInfo.getStoreKey().equals(keyInStream)) {
            // BlobIDTransformer only exists on ambry-server and replication between servers is relying on blocking channel
            // which is still using java ByteBuffer. So, no need to consider releasing stuff.
            // @todo, when netty Bytebuf is adopted for blocking channel on ambry-server, remember to release this ByteBuf.
            PutMessageFormatInputStream transformedStream = new PutMessageFormatInputStream(keyInStream, encryptionKey, props, metadata, new ByteBufInputStream(blobData.content(), true), blobData.getSize(), blobData.getBlobType(), msgInfo.getLifeVersion());
            MessageInfo transformedMsgInfo = new MessageInfo.Builder(msgInfo).size(transformedStream.getSize()).isDeleted(false).isUndeleted(false).build();
            transformationOutput = new TransformationOutput(new Message(transformedMsgInfo, transformedStream));
        } else {
            throw new IllegalStateException("StoreKey in stream: " + keyInStream + " failed to match store key from Index: " + msgInfo.getStoreKey());
        }
    } catch (Exception e) {
        transformationOutput = new TransformationOutput(e);
    }
    return transformationOutput;
}

Example 9 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.

public static void addPutMessagesToReplicasOfPartition(List<StoreKey> ids, List<Transformer> transformPerId, List<MockHost> hosts) throws MessageFormatException, IOException {
    Iterator<Transformer> transformerIterator = transformPerId.iterator();
    for (StoreKey storeKey : ids) {
        Transformer transformer = transformerIterator.next();
        BlobId id = (BlobId) storeKey;
        PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, id.getAccountId(), id.getContainerId(), BlobId.isEncrypted(id.toString()));
        MessageInfo msgInfo = msgInfoAndBuffer.messageInfo;
        ByteBuffer byteBuffer = msgInfoAndBuffer.byteBuffer;
        if (transformer != null) {
            Message message = new Message(msgInfo, new ByteBufferInputStream(byteBuffer));
            TransformationOutput output = transformer.transform(message);
            assertNull(output.getException());
            message = output.getMsg();
            byteBuffer = ByteBuffer.wrap(Utils.readBytesFromStream(message.getStream(), (int) message.getMessageInfo().getSize()));
            msgInfo = message.getMessageInfo();
        }
        for (MockHost host : hosts) {
            host.addMessage(id.getPartition(), msgInfo, byteBuffer.duplicate());
        }
    }
}

Example 10 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformerTest method testWarmup.

/**
 * Tests BlobIdTransformer's warmup() method
 * @throws Exception
 */
@Test
public void testWarmup() throws Exception {
    InputAndExpected inputAndExpected = new InputAndExpected(pairList.get(0), VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], true);
    BlobIdTransformer transformer = new BlobIdTransformer(blobIdFactory, factory.getStoreKeyConverter());
    TransformationOutput output = transformer.transform(inputAndExpected.getInput());
    Assert.assertTrue("Should lead to IllegalStateException", output.getException() instanceof IllegalStateException);
    transformer.warmup(Collections.singletonList(inputAndExpected.getInput().getMessageInfo()));
    output = transformer.transform(inputAndExpected.getInput());
    assertNull(output.getException());
    verifyOutput(output.getMsg(), inputAndExpected.getExpected());
}