Examples with TransformationOutput com.github.ambry.store.TransformationOutput used on opensource projects

Example 1 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class MessageSievingInputStream method validateAndTransform.

/**
 * Validates and potentially transforms the given input stream consisting of message data. It does so using the list
 * of {@link Transformer}s associated with this instance.
 * message corruption and acceptable formats.
 * @param inMsg the original {@link Message} that needs to be validated and possibly transformed.
 * @param msgStreamList the output list to which the sieved stream output are to be added to.
 * @param msgOffset the offset of the message in the stream.
 * @throws IOException if an exception was encountered reading or writing bytes to/from streams.
 */
private void validateAndTransform(Message inMsg, List<InputStream> msgStreamList, int msgOffset) throws IOException {
    if (transformers == null || transformers.isEmpty()) {
        // Write the message without any transformations.
        sievedMessageInfoList.add(inMsg.getMessageInfo());
        msgStreamList.add(inMsg.getStream());
    } else {
        long sieveStartTime = SystemTime.getInstance().milliseconds();
        Message msg = inMsg;
        TransformationOutput output = null;
        for (Transformer transformer : transformers) {
            output = transformer.transform(msg);
            if (output.getException() != null || output.getMsg() == null) {
                break;
            } else {
                msg = output.getMsg();
            }
        }
        if (output.getException() != null) {
            if (output.getException() instanceof MessageFormatException) {
                logger.error("Error validating/transforming the message at {} with messageInfo {} and hence skipping the message", msgOffset, inMsg.getMessageInfo(), output.getException());
                hasInvalidMessages = true;
                messageSievingCorruptMessagesDiscardedCount.inc();
            } else {
                throw new IOException("Encountered exception during transformation", output.getException());
            }
        } else if (output.getMsg() == null) {
            logger.trace("Transformation is on, and the message with id {} does not have a replacement and was discarded.", inMsg.getMessageInfo().getStoreKey());
            hasDeprecatedMessages = true;
            messageSievingDeprecatedMessagesDiscardedCount.inc();
        } else {
            MessageInfo tfmMsgInfo = output.getMsg().getMessageInfo();
            sievedMessageInfoList.add(tfmMsgInfo);
            msgStreamList.add(output.getMsg().getStream());
            logger.trace("Original message length {}, transformed bytes read {}", inMsg.getMessageInfo().getSize(), tfmMsgInfo.getSize());
        }
        singleMessageSieveTime.update(SystemTime.getInstance().milliseconds() - sieveStartTime);
    }
}

Example 2 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformerTest method testBrokenSizeMetaDataBlobOperation.

/**
 * Tests that metadata blobs with bad blob property size
 * get corrected (blob size == composite datachunk total size) in transformation
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void testBrokenSizeMetaDataBlobOperation() throws IOException, MessageFormatException {
    InputAndExpected inputAndExpected = new InputAndExpected(BLOB_ID_VERSION_1_METADATA_CONVERTED, VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], false, true, new String[] { BLOB_ID_VERSION_1_DATACHUNK_0_CONVERTED.getFirst(), BLOB_ID_VERSION_1_DATACHUNK_1_CONVERTED.getFirst() }, new String[] { BLOB_ID_VERSION_1_DATACHUNK_0_CONVERTED.getSecond(), BLOB_ID_VERSION_1_DATACHUNK_1_CONVERTED.getSecond() });
    TransformationOutput output = transformer.transform(inputAndExpected.getInput());
    assertNull("output exception should be null", output.getException());
    verifyOutput(output.getMsg(), inputAndExpected.getExpected());
}

Example 3 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformerTest method testBasicOperation.

/**
 * Tests basic use of transformer with blobs that can be converted and those that aren't
 * @throws Exception
 */
@Test
public void testBasicOperation() throws Exception {
    for (Pair pair : pairList) {
        for (Class clazz : VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS) {
            for (boolean divergeInfoFromData : new boolean[] { false, true }) {
                InputAndExpected inputAndExpected = new InputAndExpected(pair, clazz, divergeInfoFromData);
                TransformationOutput output = transformer.transform(inputAndExpected.getInput());
                assertNull("output exception should be null", output.getException());
                verifyOutput(output.getMsg(), inputAndExpected.getExpected());
            }
        }
    }
}

Example 4 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformerTest method testBrokenStoreKeyConverter.

/**
 * Tests transformer when the underlying StoreKeyConverter isn't working
 * @throws Exception
 */
@Test
public void testBrokenStoreKeyConverter() throws Exception {
    InputAndExpected inputAndExpected = new InputAndExpected(pairList.get(0), VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], false);
    TransformationOutput output = transformer.transform(inputAndExpected.getInput());
    verifyOutput(output.getMsg(), inputAndExpected.getExpected());
    factory.setException(new BlobIdTransformerTestException());
    inputAndExpected = new InputAndExpected(pairList.get(1), VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], false);
    output = transformer.transform(inputAndExpected.getInput());
    Assert.assertTrue("Should lead to IllegalStateException", output.getException() instanceof IllegalStateException);
    factory.setException(null);
    inputAndExpected = new InputAndExpected(pairList.get(2), VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], false);
    output = transformer.transform(inputAndExpected.getInput());
    verifyOutput(output.getMsg(), inputAndExpected.getExpected());
}

Example 5 with TransformationOutput

use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.

the class BlobIdTransformerTest method testMetaDataBlobOperation.

/**
 * Tests metadata blob transformation
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void testMetaDataBlobOperation() throws IOException, MessageFormatException {
    InputAndExpected inputAndExpected = new InputAndExpected(BLOB_ID_VERSION_1_METADATA_CONVERTED, VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], false, new String[] { BLOB_ID_VERSION_1_DATACHUNK_0_CONVERTED.getFirst(), BLOB_ID_VERSION_1_DATACHUNK_1_CONVERTED.getFirst() }, new String[] { BLOB_ID_VERSION_1_DATACHUNK_0_CONVERTED.getSecond(), BLOB_ID_VERSION_1_DATACHUNK_1_CONVERTED.getSecond() });
    TransformationOutput output = transformer.transform(inputAndExpected.getInput());
    assertNull("output exception should be null", output.getException());
    verifyOutput(output.getMsg(), inputAndExpected.getExpected());
}