Example 16 with Crc32
use of com.github.ambry.utils.Crc32 in project ambry by linkedin.
the class MessageFormatInputStreamTest method messageFormatPutRecordsTest.
private void messageFormatPutRecordsTest(short blobVersion, BlobType blobType, short headerVersion) throws IOException, MessageFormatException {
StoreKey key = new MockId("id1");
StoreKeyFactory keyFactory = new MockIdFactory();
short accountId = Utils.getRandomShort(TestUtils.RANDOM);
short containerId = Utils.getRandomShort(TestUtils.RANDOM);
BlobProperties prop = new BlobProperties(10, "servid", accountId, containerId, false);
byte[] encryptionKey = new byte[100];
new Random().nextBytes(encryptionKey);
byte[] usermetadata = new byte[1000];
new Random().nextBytes(usermetadata);
int blobContentSize = 2000;
byte[] data = new byte[blobContentSize];
new Random().nextBytes(data);
short lifeVersion = 1;
long blobSize = -1;
MessageFormatRecord.headerVersionToUse = headerVersion;
if (blobVersion == MessageFormatRecord.Blob_Version_V1) {
blobSize = MessageFormatRecord.Blob_Format_V1.getBlobRecordSize(blobContentSize);
} else if (blobVersion == MessageFormatRecord.Blob_Version_V2 && blobType == BlobType.DataBlob) {
blobSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blobContentSize);
} else if (blobVersion == MessageFormatRecord.Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
data = byteBufferBlob.array();
blobContentSize = data.length;
blobSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blobContentSize);
}
ByteBufferInputStream stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
MessageFormatInputStream messageFormatStream = (blobVersion == MessageFormatRecord.Blob_Version_V2) ? new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType, lifeVersion) : new PutMessageFormatBlobV1InputStream(key, prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType);
int headerSize = MessageFormatRecord.getHeaderSizeForVersion(headerVersion);
int blobEncryptionKeySize = headerVersion != MessageFormatRecord.Message_Header_Version_V1 ? MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(ByteBuffer.wrap(encryptionKey)) : 0;
int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(prop);
int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(ByteBuffer.wrap(usermetadata));
Assert.assertEquals(messageFormatStream.getSize(), headerSize + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize + key.sizeInBytes());
// verify header
byte[] headerOutput = new byte[headerSize];
messageFormatStream.read(headerOutput);
ByteBuffer headerBuf = ByteBuffer.wrap(headerOutput);
Assert.assertEquals(headerVersion, headerBuf.getShort());
if (headerVersion == MessageFormatRecord.Message_Header_Version_V3) {
Assert.assertEquals(lifeVersion, headerBuf.getShort());
}
Assert.assertEquals(blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize, headerBuf.getLong());
switch(headerVersion) {
case MessageFormatRecord.Message_Header_Version_V1:
Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
Assert.assertEquals(headerSize + key.sizeInBytes() + blobPropertiesRecordSize, headerBuf.getInt());
Assert.assertEquals(headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize, headerBuf.getInt());
break;
default:
// case MessageFormatRecord.Message_Header_Version_V2 or V3:
Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize, headerBuf.getInt());
Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize, headerBuf.getInt());
Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize, headerBuf.getInt());
}
Crc32 crc = new Crc32();
crc.update(headerOutput, 0, headerSize - MessageFormatRecord.Crc_Size);
Assert.assertEquals(crc.getValue(), headerBuf.getLong());
// verify handle
byte[] handleOutput = new byte[key.sizeInBytes()];
ByteBuffer handleOutputBuf = ByteBuffer.wrap(handleOutput);
messageFormatStream.read(handleOutput);
byte[] dest = new byte[key.sizeInBytes()];
handleOutputBuf.get(dest);
Assert.assertArrayEquals(dest, key.toBytes());
// verify encryption key
if (headerVersion != MessageFormatRecord.Message_Header_Version_V1) {
byte[] blobEncryptionKeyOutput = new byte[blobEncryptionKeySize];
ByteBuffer blobEncryptionKeyBuf = ByteBuffer.wrap(blobEncryptionKeyOutput);
messageFormatStream.read(blobEncryptionKeyOutput);
Assert.assertEquals(blobEncryptionKeyBuf.getShort(), MessageFormatRecord.Blob_Encryption_Key_V1);
Assert.assertEquals(blobEncryptionKeyBuf.getInt(), 100);
dest = new byte[100];
blobEncryptionKeyBuf.get(dest);
Assert.assertArrayEquals(dest, encryptionKey);
crc = new Crc32();
crc.update(blobEncryptionKeyOutput, 0, blobEncryptionKeySize - MessageFormatRecord.Crc_Size);
Assert.assertEquals(crc.getValue(), blobEncryptionKeyBuf.getLong());
}
// verify blob properties
byte[] blobPropertiesOutput = new byte[blobPropertiesRecordSize];
ByteBuffer blobPropertiesBuf = ByteBuffer.wrap(blobPropertiesOutput);
messageFormatStream.read(blobPropertiesOutput);
Assert.assertEquals(blobPropertiesBuf.getShort(), 1);
BlobProperties propOutput = BlobPropertiesSerDe.getBlobPropertiesFromStream(new DataInputStream(new ByteBufferInputStream(blobPropertiesBuf)));
Assert.assertEquals(10, propOutput.getBlobSize());
Assert.assertEquals("servid", propOutput.getServiceId());
Assert.assertEquals("AccountId mismatch", accountId, propOutput.getAccountId());
Assert.assertEquals("ContainerId mismatch", containerId, propOutput.getContainerId());
crc = new Crc32();
crc.update(blobPropertiesOutput, 0, blobPropertiesRecordSize - MessageFormatRecord.Crc_Size);
Assert.assertEquals(crc.getValue(), blobPropertiesBuf.getLong());
// verify user metadata
byte[] userMetadataOutput = new byte[userMetadataSize];
ByteBuffer userMetadataBuf = ByteBuffer.wrap(userMetadataOutput);
messageFormatStream.read(userMetadataOutput);
Assert.assertEquals(userMetadataBuf.getShort(), 1);
Assert.assertEquals(userMetadataBuf.getInt(), 1000);
dest = new byte[1000];
userMetadataBuf.get(dest);
Assert.assertArrayEquals(dest, usermetadata);
crc = new Crc32();
crc.update(userMetadataOutput, 0, userMetadataSize - MessageFormatRecord.Crc_Size);
Assert.assertEquals(crc.getValue(), userMetadataBuf.getLong());
// verify blob
CrcInputStream crcstream = new CrcInputStream(messageFormatStream);
DataInputStream streamData = new DataInputStream(crcstream);
Assert.assertEquals(streamData.readShort(), blobVersion);
if (blobVersion == MessageFormatRecord.Blob_Version_V2) {
Assert.assertEquals(streamData.readShort(), blobType.ordinal());
}
Assert.assertEquals(streamData.readLong(), blobContentSize);
for (int i = 0; i < blobContentSize; i++) {
Assert.assertEquals((byte) streamData.read(), data[i]);
}
long crcVal = crcstream.getValue();
Assert.assertEquals(crcVal, streamData.readLong());
// Verify Blob All
stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
messageFormatStream = (blobVersion == MessageFormatRecord.Blob_Version_V2) ? new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key, prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType);
int totalSize;
switch(headerVersion) {
case MessageFormatRecord.Message_Header_Version_V1:
totalSize = headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize + (int) blobSize;
break;
default:
// case MessageFormatRecord.Message_Header_Version_V2:
totalSize = headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) blobSize;
}
ByteBuffer allBuf = ByteBuffer.allocate(totalSize);
messageFormatStream.read(allBuf.array());
BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(new ByteBufferInputStream(allBuf), keyFactory);
Assert.assertEquals(key, blobAll.getStoreKey());
Assert.assertArrayEquals(usermetadata, blobAll.getBlobInfo().getUserMetadata());
Assert.assertEquals(blobContentSize, blobAll.getBlobData().getSize());
Assert.assertEquals(blobType, blobAll.getBlobData().getBlobType());
if (headerVersion != MessageFormatRecord.Message_Header_Version_V1) {
Assert.assertEquals(ByteBuffer.wrap(encryptionKey), blobAll.getBlobEncryptionKey());
} else {
Assert.assertEquals(null, blobAll.getBlobEncryptionKey());
}
ByteBuf byteBuf = blobAll.getBlobData().content();
try {
Assert.assertEquals(Unpooled.wrappedBuffer(data), byteBuf);
} finally {
byteBuf.release();
}
}
Also used :
ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream)
MockId(com.github.ambry.store.MockId)
DataInputStream(java.io.DataInputStream)
ByteBuf(io.netty.buffer.ByteBuf)
StoreKey(com.github.ambry.store.StoreKey)
ByteBuffer(java.nio.ByteBuffer)
StoreKeyFactory(com.github.ambry.store.StoreKeyFactory)
CrcInputStream(com.github.ambry.utils.CrcInputStream)
Random(java.util.Random)
MockIdFactory(com.github.ambry.store.MockIdFactory)
Crc32(com.github.ambry.utils.Crc32)
Example 17 with Crc32
use of com.github.ambry.utils.Crc32 in project ambry by linkedin.
the class MessageFormatRecordTest method getBlobRecordV2.
/**
* Serializes the blob content using BlobRecord Verison 2 with the passsed in params
* De-serialized the blob returns the {@link BlobData} for the same
* @param blobSize
* @param blobType
* @param blobContent
* @param outputBuffer
* @return
* @throws IOException
* @throws MessageFormatException
*/
private BlobData getBlobRecordV2(int blobSize, BlobType blobType, ByteBuffer blobContent, ByteBuffer outputBuffer) throws IOException, MessageFormatException {
MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(outputBuffer, blobSize, blobType);
outputBuffer.put(blobContent);
Crc32 crc = new Crc32();
crc.update(outputBuffer.array(), 0, outputBuffer.position());
outputBuffer.putLong(crc.getValue());
outputBuffer.flip();
return MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(outputBuffer));
}
Also used :
Crc32(com.github.ambry.utils.Crc32)
ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream)
Example 18 with Crc32
use of com.github.ambry.utils.Crc32 in project ambry by linkedin.
the class MessageFormatRecordTest method deserializeTest.
// TODO Separate this mega test into smaller tests
@Test
public void deserializeTest() throws MessageFormatException, IOException {
{
// Test message header V1
ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize());
MessageFormatRecord.MessageHeader_Format_V1.serializeHeader(header, 1000, 10, -1, 20, 30);
header.flip();
MessageFormatRecord.MessageHeader_Format_V1 format = new MessageFormatRecord.MessageHeader_Format_V1(header);
Assert.assertEquals(format.getMessageSize(), 1000);
Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
// corrupt message header V1
header.put(10, (byte) 1);
format = new MessageFormatRecord.MessageHeader_Format_V1(header);
try {
format.verifyHeader();
Assert.assertEquals(true, false);
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
}
{
// Test message header V2
ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize());
MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(header, 1000, 5, 10, -1, 20, 30);
header.flip();
MessageFormatRecord.MessageHeader_Format_V2 format = new MessageFormatRecord.MessageHeader_Format_V2(header);
Assert.assertEquals(format.getMessageSize(), 1000);
Assert.assertEquals(format.getBlobEncryptionKeyRecordRelativeOffset(), 5);
Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
// corrupt message header V2
header.put(10, (byte) 1);
format = new MessageFormatRecord.MessageHeader_Format_V2(header);
try {
format.verifyHeader();
fail("Corrupt header verification should have failed");
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
}
{
// Test message header V3
ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V3.getHeaderSize());
MessageFormatRecord.MessageHeader_Format_V3.serializeHeader(header, (short) 2, 1000, 5, 10, -1, 20, 30);
header.flip();
MessageFormatRecord.MessageHeader_Format_V3 format = new MessageFormatRecord.MessageHeader_Format_V3(header);
Assert.assertEquals(format.getMessageSize(), 1000);
Assert.assertEquals(format.getLifeVersion(), (short) 2);
Assert.assertEquals(format.getBlobEncryptionKeyRecordRelativeOffset(), 5);
Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
// corrupt message header V3
header.put(10, (byte) 1);
format = new MessageFormatRecord.MessageHeader_Format_V3(header);
try {
format.verifyHeader();
fail("Corrupt header verification should have failed");
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
}
// Test blob encryption key record
ByteBuffer blobEncryptionKey = ByteBuffer.allocate(1000);
new Random().nextBytes(blobEncryptionKey.array());
ByteBuffer output = ByteBuffer.allocate(MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey));
MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(output, blobEncryptionKey);
output.flip();
ByteBuffer bufOutput = MessageFormatRecord.deserializeBlobEncryptionKey(new ByteBufferInputStream(output));
Assert.assertArrayEquals(blobEncryptionKey.array(), bufOutput.array());
// Corrupt encryption key record
output.flip();
Byte currentRandomByte = output.get(10);
output.put(10, (byte) (currentRandomByte + 1));
try {
MessageFormatRecord.deserializeBlobEncryptionKey(new ByteBufferInputStream(output));
fail("Encryption key record deserialization should have failed for corrupt data");
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
// Test usermetadata V1 record
ByteBuffer usermetadata = ByteBuffer.allocate(1000);
new Random().nextBytes(usermetadata.array());
output = ByteBuffer.allocate(MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(usermetadata));
MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(output, usermetadata);
output.flip();
bufOutput = MessageFormatRecord.deserializeUserMetadata(new ByteBufferInputStream(output));
Assert.assertArrayEquals(usermetadata.array(), bufOutput.array());
// corrupt usermetadata record V1
output.flip();
currentRandomByte = output.get(10);
output.put(10, (byte) (currentRandomByte + 1));
try {
MessageFormatRecord.deserializeUserMetadata(new ByteBufferInputStream(output));
Assert.assertEquals(true, false);
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
// Test blob record V1
ByteBuffer data = ByteBuffer.allocate(2000);
new Random().nextBytes(data.array());
long size = MessageFormatRecord.Blob_Format_V1.getBlobRecordSize(2000);
ByteBuffer sData = ByteBuffer.allocate((int) size);
MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(sData, 2000);
sData.put(data);
Crc32 crc = new Crc32();
crc.update(sData.array(), 0, sData.position());
sData.putLong(crc.getValue());
sData.flip();
BlobData blobData = MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(sData));
Assert.assertEquals(blobData.getSize(), 2000);
byte[] verify = new byte[2000];
blobData.content().readBytes(verify);
Assert.assertArrayEquals(verify, data.array());
blobData.release();
// corrupt blob record V1
sData.flip();
currentRandomByte = sData.get(10);
sData.put(10, (byte) (currentRandomByte + 1));
try {
MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(sData));
Assert.assertEquals(true, false);
} catch (MessageFormatException e) {
Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
}
}
Also used :
ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream)
ByteBuffer(java.nio.ByteBuffer)
Random(java.util.Random)
ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom)
Crc32(com.github.ambry.utils.Crc32)
MessageFormatRecord(com.github.ambry.messageformat.MessageFormatRecord)
Test(org.junit.Test)
Example 19 with Crc32
use of com.github.ambry.utils.Crc32 in project ambry by linkedin.
the class StoredBlob method makeGetResponse.
/**
* Make a {@link GetResponse} for the given {@link GetRequest} for which the given {@link ServerErrorCode} was
* encountered. The request could be for BlobInfo or for Blob (the only two options that the router would request
* for).
* @param getRequest the {@link GetRequest} for which the response is being constructed.
* @param getError the {@link ServerErrorCode} that was encountered.
* @return the constructed {@link GetResponse}
* @throws IOException if there was an error constructing the response.
*/
GetResponse makeGetResponse(GetRequest getRequest, ServerErrorCode getError) throws IOException {
GetResponse getResponse;
if (getError == ServerErrorCode.No_Error) {
List<PartitionRequestInfo> infos = getRequest.getPartitionInfoList();
if (infos.size() != 1 || infos.get(0).getBlobIds().size() != 1) {
getError = ServerErrorCode.Unknown_Error;
}
}
ServerErrorCode serverError;
ServerErrorCode partitionError;
boolean isDataBlob = false;
try {
String id = getRequest.getPartitionInfoList().get(0).getBlobIds().get(0).getID();
isDataBlob = blobs.get(id).type == BlobType.DataBlob;
} catch (Exception ignored) {
}
if (!getErrorOnDataBlobOnly || isDataBlob) {
// set it in the partitionResponseInfo
if (getError == ServerErrorCode.No_Error || getError == ServerErrorCode.Blob_Expired || getError == ServerErrorCode.Blob_Deleted || getError == ServerErrorCode.Blob_Not_Found || getError == ServerErrorCode.Blob_Authorization_Failure || getError == ServerErrorCode.Disk_Unavailable) {
partitionError = getError;
serverError = ServerErrorCode.No_Error;
} else {
serverError = getError;
// does not matter - this will not be checked if serverError is not No_Error.
partitionError = ServerErrorCode.No_Error;
}
} else {
serverError = ServerErrorCode.No_Error;
partitionError = ServerErrorCode.No_Error;
}
if (serverError == ServerErrorCode.No_Error) {
int byteBufferSize;
ByteBuffer byteBuffer;
StoreKey key = getRequest.getPartitionInfoList().get(0).getBlobIds().get(0);
short accountId = Account.UNKNOWN_ACCOUNT_ID;
short containerId = Container.UNKNOWN_CONTAINER_ID;
long operationTimeMs = Utils.Infinite_Time;
StoredBlob blob = blobs.get(key.getID());
ServerErrorCode processedError = errorForGet(key.getID(), blob, getRequest);
MessageMetadata msgMetadata = null;
if (processedError == ServerErrorCode.No_Error) {
ByteBuffer buf = blobs.get(key.getID()).serializedSentPutRequest.duplicate();
// read off the size
buf.getLong();
// read off the type.
buf.getShort();
PutRequest originalBlobPutReq = PutRequest.readFrom(new DataInputStream(new ByteBufferInputStream(buf)), clusterMap);
switch(getRequest.getMessageFormatFlag()) {
case BlobInfo:
BlobProperties blobProperties = originalBlobPutReq.getBlobProperties();
accountId = blobProperties.getAccountId();
containerId = blobProperties.getContainerId();
operationTimeMs = blobProperties.getCreationTimeInMs();
ByteBuffer userMetadata = originalBlobPutReq.getUsermetadata();
byteBufferSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties) + MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
byteBuffer = ByteBuffer.allocate(byteBufferSize);
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(byteBuffer, blobProperties);
MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(byteBuffer, userMetadata);
break;
case Blob:
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
byteBufferSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
byteBuffer = ByteBuffer.allocate(byteBufferSize);
MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize(), originalBlobPutReq.getBlobType());
break;
case MessageFormatRecord.Blob_Version_V1:
byteBufferSize = (int) MessageFormatRecord.Blob_Format_V1.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
byteBuffer = ByteBuffer.allocate(byteBufferSize);
MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBuffer.put(Utils.readBytesFromStream(originalBlobPutReq.getBlobStream(), (int) originalBlobPutReq.getBlobSize()));
Crc32 crc = new Crc32();
crc.update(byteBuffer.array(), 0, byteBuffer.position());
byteBuffer.putLong(crc.getValue());
break;
case All:
blobProperties = originalBlobPutReq.getBlobProperties();
accountId = blobProperties.getAccountId();
containerId = blobProperties.getContainerId();
userMetadata = originalBlobPutReq.getUsermetadata();
operationTimeMs = originalBlobPutReq.getBlobProperties().getCreationTimeInMs();
int blobHeaderSize = MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize();
int blobEncryptionRecordSize = originalBlobPutReq.getBlobEncryptionKey() != null ? MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(originalBlobPutReq.getBlobEncryptionKey().duplicate()) : 0;
int blobPropertiesSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
int blobInfoSize = blobPropertiesSize + userMetadataSize;
int blobRecordSize;
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
blobRecordSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
break;
case MessageFormatRecord.Blob_Version_V1:
blobRecordSize = (int) MessageFormatRecord.Blob_Format_V1.getBlobRecordSize((int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBufferSize = blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobInfoSize + blobRecordSize;
byteBuffer = ByteBuffer.allocate(byteBufferSize);
try {
MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(byteBuffer, blobEncryptionRecordSize + blobInfoSize + blobRecordSize, originalBlobPutReq.getBlobEncryptionKey() == null ? Message_Header_Invalid_Relative_Offset : blobHeaderSize + key.sizeInBytes(), blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize, Message_Header_Invalid_Relative_Offset, blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobPropertiesSize, blobHeaderSize + key.sizeInBytes() + blobEncryptionRecordSize + blobInfoSize);
} catch (MessageFormatException e) {
e.printStackTrace();
}
byteBuffer.put(key.toBytes());
if (originalBlobPutReq.getBlobEncryptionKey() != null) {
MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(byteBuffer, originalBlobPutReq.getBlobEncryptionKey().duplicate());
msgMetadata = new MessageMetadata(originalBlobPutReq.getBlobEncryptionKey().duplicate());
}
MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(byteBuffer, blobProperties);
MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(byteBuffer, userMetadata);
int blobRecordStart = byteBuffer.position();
switch(blobFormatVersion) {
case MessageFormatRecord.Blob_Version_V2:
MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize(), originalBlobPutReq.getBlobType());
break;
case MessageFormatRecord.Blob_Version_V1:
MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(byteBuffer, (int) originalBlobPutReq.getBlobSize());
break;
default:
throw new IllegalStateException("Blob format version " + blobFormatVersion + " not supported.");
}
byteBuffer.put(Utils.readBytesFromStream(originalBlobPutReq.getBlobStream(), (int) originalBlobPutReq.getBlobSize()));
crc = new Crc32();
crc.update(byteBuffer.array(), blobRecordStart, blobRecordSize - MessageFormatRecord.Crc_Size);
byteBuffer.putLong(crc.getValue());
break;
default:
throw new IOException("GetRequest flag is not supported: " + getRequest.getMessageFormatFlag());
}
} else if (processedError == ServerErrorCode.Blob_Deleted) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Deleted;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else if (processedError == ServerErrorCode.Blob_Expired) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Expired;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else if (processedError == ServerErrorCode.Blob_Authorization_Failure) {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Authorization_Failure;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
} else {
if (partitionError == ServerErrorCode.No_Error) {
partitionError = ServerErrorCode.Blob_Not_Found;
}
byteBuffer = ByteBuffer.allocate(0);
byteBufferSize = 0;
}
byteBuffer.flip();
ByteBufferSend responseSend = new ByteBufferSend(byteBuffer);
List<MessageInfo> messageInfoList = new ArrayList<>();
List<MessageMetadata> messageMetadataList = new ArrayList<>();
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>();
if (partitionError == ServerErrorCode.No_Error) {
messageInfoList.add(new MessageInfo(key, byteBufferSize, false, blob.isTtlUpdated(), blob.isUndeleted(), blob.expiresAt, null, accountId, containerId, operationTimeMs, blob.lifeVersion));
messageMetadataList.add(msgMetadata);
}
PartitionResponseInfo partitionResponseInfo = partitionError == ServerErrorCode.No_Error ? new PartitionResponseInfo(getRequest.getPartitionInfoList().get(0).getPartition(), messageInfoList, messageMetadataList) : new PartitionResponseInfo(getRequest.getPartitionInfoList().get(0).getPartition(), partitionError);
partitionResponseInfoList.add(partitionResponseInfo);
getResponse = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, responseSend, serverError);
} else {
getResponse = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), new ArrayList<PartitionResponseInfo>(), new ByteBufferSend(ByteBuffer.allocate(0)), serverError);
}
return getResponse;
}
Also used :
ArrayList(java.util.ArrayList)
MessageMetadata(com.github.ambry.messageformat.MessageMetadata)
PartitionResponseInfo(com.github.ambry.protocol.PartitionResponseInfo)
MessageFormatException(com.github.ambry.messageformat.MessageFormatException)
ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream)
PutRequest(com.github.ambry.protocol.PutRequest)
IOException(java.io.IOException)
PartitionRequestInfo(com.github.ambry.protocol.PartitionRequestInfo)
DataInputStream(java.io.DataInputStream)
GetResponse(com.github.ambry.protocol.GetResponse)
ByteBuffer(java.nio.ByteBuffer)
StoreKey(com.github.ambry.store.StoreKey)
ServerErrorCode(com.github.ambry.server.ServerErrorCode)
IOException(java.io.IOException)
MessageFormatException(com.github.ambry.messageformat.MessageFormatException)
MessageInfo(com.github.ambry.store.MessageInfo)
Crc32(com.github.ambry.utils.Crc32)
ByteBufferSend(com.github.ambry.network.ByteBufferSend)
BlobProperties(com.github.ambry.messageformat.BlobProperties)
Example 20 with Crc32
use of com.github.ambry.utils.Crc32 in project ambry by linkedin.
the class LogSegment method writeHeader.
/**
* Writes a header describing the segment.
* @param capacityInBytes the intended capacity of the segment.
* @throws StoreException if there is any store exception while writing to the file.
*/
private void writeHeader(long capacityInBytes) throws StoreException {
Crc32 crc = new Crc32();
ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
buffer.putShort(VERSION);
buffer.putLong(capacityInBytes);
crc.update(buffer.array(), 0, HEADER_SIZE - CRC_SIZE);
buffer.putLong(crc.getValue());
buffer.flip();
appendFrom(buffer);
}
Also used :
Crc32(com.github.ambry.utils.Crc32)
ByteBuffer(java.nio.ByteBuffer)
Aggregations
Crc32 (com.github.ambry.utils.Crc32)22
ByteBuffer (java.nio.ByteBuffer)14
ByteBufferInputStream (com.github.ambry.utils.ByteBufferInputStream)6
CrcInputStream (com.github.ambry.utils.CrcInputStream)6
Random (java.util.Random)6
StoreKey (com.github.ambry.store.StoreKey)5
DataInputStream (java.io.DataInputStream)4
Test (org.junit.Test)4
MockId (com.github.ambry.store.MockId)3
HashMap (java.util.HashMap)3
MockIdFactory (com.github.ambry.store.MockIdFactory)2
StoreKeyFactory (com.github.ambry.store.StoreKeyFactory)2
ArrayList (java.util.ArrayList)2
Map (java.util.Map)2
MetricRegistry (com.codahale.metrics.MetricRegistry)1
BlobProperties (com.github.ambry.messageformat.BlobProperties)1
MessageFormatException (com.github.ambry.messageformat.MessageFormatException)1
MessageFormatRecord (com.github.ambry.messageformat.MessageFormatRecord)1
MessageMetadata (com.github.ambry.messageformat.MessageMetadata)1
ByteBufferSend (com.github.ambry.network.ByteBufferSend)1
