Examples with ByteBufferInputStream com.github.ambry.utils.ByteBufferInputStream used on opensource projects

Example 56 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageFormatWriteSetTest method writeSetTest.

@Test
public void writeSetTest() throws IOException, StoreException {
    byte[] buf = new byte[2000];
    MessageInfo info1 = new MessageInfo(new MockId("id1"), 1000, 123, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt());
    MessageInfo info2 = new MessageInfo(new MockId("id2"), 1000, 123, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt());
    List<MessageInfo> infoList = new ArrayList<MessageInfo>();
    infoList.add(info1);
    infoList.add(info2);
    ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(ByteBuffer.wrap(buf));
    MessageFormatWriteSet set = new MessageFormatWriteSet(byteBufferInputStream, infoList, false);
    MockWrite write = new MockWrite(2000);
    long written = set.writeTo(write);
    Assert.assertEquals(written, 2000);
    Assert.assertEquals(write.getBuffer().limit(), 2000);
    Assert.assertArrayEquals(write.getBuffer().array(), buf);
}

Example 57 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageFormatRecordTest method getBlobRecordV2.

/**
 * Serializes the blob content using BlobRecord Verison 2 with the passsed in params
 * De-serialized the blob returns the {@link BlobData} for the same
 * @param blobSize
 * @param blobType
 * @param blobContent
 * @param outputBuffer
 * @return
 * @throws IOException
 * @throws MessageFormatException
 */
private BlobData getBlobRecordV2(int blobSize, BlobType blobType, ByteBuffer blobContent, ByteBuffer outputBuffer) throws IOException, MessageFormatException {
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(outputBuffer, blobSize, blobType);
    outputBuffer.put(blobContent);
    Crc32 crc = new Crc32();
    crc.update(outputBuffer.array(), 0, outputBuffer.position());
    outputBuffer.putLong(crc.getValue());
    outputBuffer.flip();
    return MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(outputBuffer));
}

Example 58 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageFormatRecordTest method deserializeTest.

// TODO Separate this mega test into smaller tests
@Test
public void deserializeTest() throws MessageFormatException, IOException {
    {
        // Test message header V1
        ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize());
        MessageFormatRecord.MessageHeader_Format_V1.serializeHeader(header, 1000, 10, -1, 20, 30);
        header.flip();
        MessageFormatRecord.MessageHeader_Format_V1 format = new MessageFormatRecord.MessageHeader_Format_V1(header);
        Assert.assertEquals(format.getMessageSize(), 1000);
        Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
        Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
        Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
        // corrupt message header V1
        header.put(10, (byte) 1);
        format = new MessageFormatRecord.MessageHeader_Format_V1(header);
        try {
            format.verifyHeader();
            Assert.assertEquals(true, false);
        } catch (MessageFormatException e) {
            Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
        }
    }
    {
        // Test message header V2
        ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize());
        MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(header, 1000, 5, 10, -1, 20, 30);
        header.flip();
        MessageFormatRecord.MessageHeader_Format_V2 format = new MessageFormatRecord.MessageHeader_Format_V2(header);
        Assert.assertEquals(format.getMessageSize(), 1000);
        Assert.assertEquals(format.getBlobEncryptionKeyRecordRelativeOffset(), 5);
        Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
        Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
        Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
        // corrupt message header V2
        header.put(10, (byte) 1);
        format = new MessageFormatRecord.MessageHeader_Format_V2(header);
        try {
            format.verifyHeader();
            fail("Corrupt header verification should have failed");
        } catch (MessageFormatException e) {
            Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
        }
    }
    {
        // Test message header V3
        ByteBuffer header = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V3.getHeaderSize());
        MessageFormatRecord.MessageHeader_Format_V3.serializeHeader(header, (short) 2, 1000, 5, 10, -1, 20, 30);
        header.flip();
        MessageFormatRecord.MessageHeader_Format_V3 format = new MessageFormatRecord.MessageHeader_Format_V3(header);
        Assert.assertEquals(format.getMessageSize(), 1000);
        Assert.assertEquals(format.getLifeVersion(), (short) 2);
        Assert.assertEquals(format.getBlobEncryptionKeyRecordRelativeOffset(), 5);
        Assert.assertEquals(format.getBlobPropertiesRecordRelativeOffset(), 10);
        Assert.assertEquals(format.getUserMetadataRecordRelativeOffset(), 20);
        Assert.assertEquals(format.getBlobRecordRelativeOffset(), 30);
        // corrupt message header V3
        header.put(10, (byte) 1);
        format = new MessageFormatRecord.MessageHeader_Format_V3(header);
        try {
            format.verifyHeader();
            fail("Corrupt header verification should have failed");
        } catch (MessageFormatException e) {
            Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
        }
    }
    // Test blob encryption key record
    ByteBuffer blobEncryptionKey = ByteBuffer.allocate(1000);
    new Random().nextBytes(blobEncryptionKey.array());
    ByteBuffer output = ByteBuffer.allocate(MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey));
    MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(output, blobEncryptionKey);
    output.flip();
    ByteBuffer bufOutput = MessageFormatRecord.deserializeBlobEncryptionKey(new ByteBufferInputStream(output));
    Assert.assertArrayEquals(blobEncryptionKey.array(), bufOutput.array());
    // Corrupt encryption key record
    output.flip();
    Byte currentRandomByte = output.get(10);
    output.put(10, (byte) (currentRandomByte + 1));
    try {
        MessageFormatRecord.deserializeBlobEncryptionKey(new ByteBufferInputStream(output));
        fail("Encryption key record deserialization should have failed for corrupt data");
    } catch (MessageFormatException e) {
        Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
    }
    // Test usermetadata V1 record
    ByteBuffer usermetadata = ByteBuffer.allocate(1000);
    new Random().nextBytes(usermetadata.array());
    output = ByteBuffer.allocate(MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(usermetadata));
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(output, usermetadata);
    output.flip();
    bufOutput = MessageFormatRecord.deserializeUserMetadata(new ByteBufferInputStream(output));
    Assert.assertArrayEquals(usermetadata.array(), bufOutput.array());
    // corrupt usermetadata record V1
    output.flip();
    currentRandomByte = output.get(10);
    output.put(10, (byte) (currentRandomByte + 1));
    try {
        MessageFormatRecord.deserializeUserMetadata(new ByteBufferInputStream(output));
        Assert.assertEquals(true, false);
    } catch (MessageFormatException e) {
        Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
    }
    // Test blob record V1
    ByteBuffer data = ByteBuffer.allocate(2000);
    new Random().nextBytes(data.array());
    long size = MessageFormatRecord.Blob_Format_V1.getBlobRecordSize(2000);
    ByteBuffer sData = ByteBuffer.allocate((int) size);
    MessageFormatRecord.Blob_Format_V1.serializePartialBlobRecord(sData, 2000);
    sData.put(data);
    Crc32 crc = new Crc32();
    crc.update(sData.array(), 0, sData.position());
    sData.putLong(crc.getValue());
    sData.flip();
    BlobData blobData = MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(sData));
    Assert.assertEquals(blobData.getSize(), 2000);
    byte[] verify = new byte[2000];
    blobData.content().readBytes(verify);
    Assert.assertArrayEquals(verify, data.array());
    blobData.release();
    // corrupt blob record V1
    sData.flip();
    currentRandomByte = sData.get(10);
    sData.put(10, (byte) (currentRandomByte + 1));
    try {
        MessageFormatRecord.deserializeBlob(new ByteBufferInputStream(sData));
        Assert.assertEquals(true, false);
    } catch (MessageFormatException e) {
        Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
    }
}

Example 59 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageFormatRecordTest method testUpdateRecordV3.

/**
 * Tests UpdateRecord V3 for serialization and deserialization
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void testUpdateRecordV3() throws IOException, MessageFormatException {
    // Test update V3 record
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long updateTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
    long updatedExpiryTimesMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
    for (SubRecord.Type type : SubRecord.Type.values()) {
        ByteBuffer updateRecordBuf = ByteBuffer.allocate(Update_Format_V3.getRecordSize(type));
        UpdateRecord updateRecord = null;
        switch(type) {
            case DELETE:
                DeleteSubRecord deleteSubRecord = new DeleteSubRecord();
                updateRecord = new UpdateRecord(accountId, containerId, updateTimeMs, deleteSubRecord);
                break;
            case TTL_UPDATE:
                TtlUpdateSubRecord ttlUpdateSubRecord = new TtlUpdateSubRecord(updatedExpiryTimesMs);
                updateRecord = new UpdateRecord(accountId, containerId, updateTimeMs, ttlUpdateSubRecord);
                break;
            case UNDELETE:
                UndeleteSubRecord undeleteSubRecord = new UndeleteSubRecord();
                updateRecord = new UpdateRecord(accountId, containerId, updateTimeMs, undeleteSubRecord);
                break;
            default:
                fail("Unknown update record type: " + type);
        }
        Update_Format_V3.serialize(updateRecordBuf, updateRecord);
        updateRecordBuf.flip();
        UpdateRecord deserializeUpdateRecord = MessageFormatRecord.deserializeUpdateRecord(new ByteBufferInputStream(updateRecordBuf));
        Assert.assertEquals("AccountId mismatch ", accountId, deserializeUpdateRecord.getAccountId());
        Assert.assertEquals("ContainerId mismatch ", containerId, deserializeUpdateRecord.getContainerId());
        Assert.assertEquals("UpdateTime mismatch ", updateTimeMs, deserializeUpdateRecord.getUpdateTimeInMs());
        Assert.assertEquals("Type of update record incorrect", type, deserializeUpdateRecord.getType());
        switch(type) {
            case DELETE:
                Assert.assertNotNull("DeleteSubRecord is null", deserializeUpdateRecord.getDeleteSubRecord());
                break;
            case TTL_UPDATE:
                TtlUpdateSubRecord ttlUpdateSubRecord = deserializeUpdateRecord.getTtlUpdateSubRecord();
                Assert.assertNotNull("TtlUpdateSubRecord is null", ttlUpdateSubRecord);
                Assert.assertEquals("Updated expiry time is incorrect", updatedExpiryTimesMs, ttlUpdateSubRecord.getUpdatedExpiryTimeMs());
                break;
            case UNDELETE:
                Assert.assertNotNull("UndeleteSubRecord is null", deserializeUpdateRecord.getUndeleteSubRecord());
                break;
            default:
                fail("Unknown update record type: " + type);
        }
        // corrupt update V3 record
        updateRecordBuf.flip();
        byte toCorrupt = updateRecordBuf.get(10);
        updateRecordBuf.put(10, (byte) (toCorrupt + 1));
        try {
            MessageFormatRecord.deserializeUpdateRecord(new ByteBufferInputStream(updateRecordBuf));
            fail("Deserialization of a corrupt update record V3 should have failed ");
        } catch (MessageFormatException e) {
            Assert.assertEquals(e.getErrorCode(), MessageFormatErrorCodes.Data_Corrupt);
        }
    }
}

Example 60 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageFormatRecordTest method deserializeMetadataContentV2.

private CompositeBlobInfo deserializeMetadataContentV2(ByteBuffer metadataContent, StoreKeyFactory storeKeyFactory) throws MessageFormatException, IOException {
    ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(metadataContent);
    DataInputStream inputStream = new DataInputStream(byteBufferInputStream);
    short metadataContentVersion = inputStream.readShort();
    Assert.assertEquals("Metadata Content Version mismatch ", MessageFormatRecord.Metadata_Content_Version_V2, metadataContentVersion);
    return MessageFormatRecord.Metadata_Content_Format_V2.deserializeMetadataContentRecord(inputStream, storeKeyFactory);
}