use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.
the class BlobIdTransformer method transform.
@Override
public TransformationOutput transform(Message message) {
Message transformedMsg = null;
try {
Objects.requireNonNull(message, "message must not be null");
Objects.requireNonNull(message.getMessageInfo(), "message's messageInfo must not be null");
Objects.requireNonNull(message.getStream(), "message's inputStream must not be null");
StoreKey oldStoreKey = message.getMessageInfo().getStoreKey();
StoreKey newStoreKey = storeKeyConverter.getConverted(oldStoreKey);
if (newStoreKey != null) {
transformedMsg = newMessage(message.getStream(), newStoreKey, message.getMessageInfo());
}
} catch (Exception e) {
return new TransformationOutput(e);
}
return new TransformationOutput(transformedMsg);
}
use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.
the class ValidatingKeyConvertingTransformer method transform.
@Override
public TransformationOutput transform(Message message) {
ByteBuffer encryptionKey;
BlobProperties props;
ByteBuffer metadata;
BlobData blobData;
MessageInfo msgInfo = message.getMessageInfo();
InputStream msgStream = message.getStream();
TransformationOutput transformationOutput;
try {
// Read header
ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
msgStream.read(headerVersion.array());
short version = headerVersion.getShort();
if (!isValidHeaderVersion(version)) {
throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
}
int headerSize = getHeaderSizeForVersion(version);
ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize);
headerBuffer.put(headerVersion.array());
msgStream.read(headerBuffer.array(), Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
headerBuffer.rewind();
MessageHeader_Format header = getMessageHeader(version, headerBuffer);
header.verifyHeader();
StoreKey originalKey = storeKeyFactory.getStoreKey(new DataInputStream(msgStream));
if (header.isPutRecord()) {
encryptionKey = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(msgStream) : null;
props = deserializeBlobProperties(msgStream);
metadata = deserializeUserMetadata(msgStream);
blobData = deserializeBlob(msgStream);
} else {
throw new IllegalArgumentException("Message cannot be a deleted record ");
}
if (msgInfo.getStoreKey().equals(originalKey)) {
StoreKey newKey = storeKeyConverter.convert(Collections.singletonList(originalKey)).get(originalKey);
if (newKey == null) {
System.out.println("No mapping for the given key, transformed message will be null");
transformationOutput = new TransformationOutput((Message) null);
} else {
MessageInfo transformedMsgInfo;
PutMessageFormatInputStream transformedStream = new PutMessageFormatInputStream(newKey, encryptionKey, props, metadata, new ByteBufInputStream(blobData.content(), true), blobData.getSize(), blobData.getBlobType(), msgInfo.getLifeVersion());
transformedMsgInfo = new MessageInfo.Builder(msgInfo).storeKey(newKey).size(transformedStream.getSize()).isUndeleted(false).build();
transformationOutput = new TransformationOutput(new Message(transformedMsgInfo, transformedStream));
}
} else {
throw new IllegalStateException("StoreKey in log " + originalKey + " failed to match store key from Index " + msgInfo.getStoreKey());
}
} catch (Exception e) {
transformationOutput = new TransformationOutput(e);
}
return transformationOutput;
}
use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.
the class ValidatingTransformer method transform.
@Override
public TransformationOutput transform(Message message) {
ByteBuffer encryptionKey;
BlobProperties props;
ByteBuffer metadata;
BlobData blobData;
MessageInfo msgInfo = message.getMessageInfo();
InputStream msgStream = message.getStream();
TransformationOutput transformationOutput = null;
try {
// Read header
ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
msgStream.read(headerVersion.array());
short version = headerVersion.getShort();
if (!isValidHeaderVersion(version)) {
throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
}
int headerSize = getHeaderSizeForVersion(version);
ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize);
headerBuffer.put(headerVersion.array());
msgStream.read(headerBuffer.array(), Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
headerBuffer.rewind();
MessageHeader_Format header = getMessageHeader(version, headerBuffer);
header.verifyHeader();
StoreKey keyInStream = storeKeyFactory.getStoreKey(new DataInputStream(msgStream));
if (header.isPutRecord()) {
if (header.hasLifeVersion() && header.getLifeVersion() != msgInfo.getLifeVersion()) {
logger.trace("LifeVersion in stream: {} failed to match lifeVersion from Index: {} for key {}", header.getLifeVersion(), msgInfo.getLifeVersion(), keyInStream);
}
encryptionKey = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(msgStream) : null;
props = deserializeBlobProperties(msgStream);
metadata = deserializeUserMetadata(msgStream);
blobData = deserializeBlob(msgStream);
} else {
throw new IllegalStateException("Message cannot be anything rather than put record ");
}
if (msgInfo.getStoreKey().equals(keyInStream)) {
// BlobIDTransformer only exists on ambry-server and replication between servers is relying on blocking channel
// which is still using java ByteBuffer. So, no need to consider releasing stuff.
// @todo, when netty Bytebuf is adopted for blocking channel on ambry-server, remember to release this ByteBuf.
PutMessageFormatInputStream transformedStream = new PutMessageFormatInputStream(keyInStream, encryptionKey, props, metadata, new ByteBufInputStream(blobData.content(), true), blobData.getSize(), blobData.getBlobType(), msgInfo.getLifeVersion());
MessageInfo transformedMsgInfo = new MessageInfo.Builder(msgInfo).size(transformedStream.getSize()).isDeleted(false).isUndeleted(false).build();
transformationOutput = new TransformationOutput(new Message(transformedMsgInfo, transformedStream));
} else {
throw new IllegalStateException("StoreKey in stream: " + keyInStream + " failed to match store key from Index: " + msgInfo.getStoreKey());
}
} catch (Exception e) {
transformationOutput = new TransformationOutput(e);
}
return transformationOutput;
}
use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.
the class ReplicationTestHelper method addPutMessagesToReplicasOfPartition.
public static void addPutMessagesToReplicasOfPartition(List<StoreKey> ids, List<Transformer> transformPerId, List<MockHost> hosts) throws MessageFormatException, IOException {
Iterator<Transformer> transformerIterator = transformPerId.iterator();
for (StoreKey storeKey : ids) {
Transformer transformer = transformerIterator.next();
BlobId id = (BlobId) storeKey;
PutMsgInfoAndBuffer msgInfoAndBuffer = createPutMessage(id, id.getAccountId(), id.getContainerId(), BlobId.isEncrypted(id.toString()));
MessageInfo msgInfo = msgInfoAndBuffer.messageInfo;
ByteBuffer byteBuffer = msgInfoAndBuffer.byteBuffer;
if (transformer != null) {
Message message = new Message(msgInfo, new ByteBufferInputStream(byteBuffer));
TransformationOutput output = transformer.transform(message);
assertNull(output.getException());
message = output.getMsg();
byteBuffer = ByteBuffer.wrap(Utils.readBytesFromStream(message.getStream(), (int) message.getMessageInfo().getSize()));
msgInfo = message.getMessageInfo();
}
for (MockHost host : hosts) {
host.addMessage(id.getPartition(), msgInfo, byteBuffer.duplicate());
}
}
}
use of com.github.ambry.store.TransformationOutput in project ambry by linkedin.
the class BlobIdTransformerTest method testWarmup.
/**
* Tests BlobIdTransformer's warmup() method
* @throws Exception
*/
@Test
public void testWarmup() throws Exception {
InputAndExpected inputAndExpected = new InputAndExpected(pairList.get(0), VALID_MESSAGE_FORMAT_INPUT_STREAM_IMPLS[0], true);
BlobIdTransformer transformer = new BlobIdTransformer(blobIdFactory, factory.getStoreKeyConverter());
TransformationOutput output = transformer.transform(inputAndExpected.getInput());
Assert.assertTrue("Should lead to IllegalStateException", output.getException() instanceof IllegalStateException);
transformer.warmup(Collections.singletonList(inputAndExpected.getInput().getMessageInfo()));
output = transformer.transform(inputAndExpected.getInput());
assertNull(output.getException());
verifyOutput(output.getMsg(), inputAndExpected.getExpected());
}
Aggregations