Examples with StoreKey com.github.ambry.store.StoreKey used on opensource projects

Example 11 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ValidatingKeyConvertingTransformer method testDeletedRecords.

private void testDeletedRecords(short blobVersion, BlobType blobType) throws Exception {
    // MessageSievingInputStream contains put records for 2 valid blobs and 1 delete record
    // id1(put record for valid blob), id2(delete record) and id3(put record for valid blob)
    ArrayList<Short> versions = new ArrayList<>();
    versions.add(Message_Header_Version_V1);
    if (blobVersion != Blob_Version_V1) {
        versions.add(Message_Header_Version_V2);
        versions.add(Message_Header_Version_V3);
    }
    try {
        for (short version : versions) {
            headerVersionToUse = version;
            // create message stream for blob 1
            StoreKey key1 = new MockId("id1");
            short accountId = Utils.getRandomShort(RANDOM);
            short containerId = Utils.getRandomShort(RANDOM);
            BlobProperties prop1 = new BlobProperties(10, "servid1", accountId, containerId, false);
            byte[] encryptionKey1 = new byte[100];
            RANDOM.nextBytes(encryptionKey1);
            byte[] usermetadata1 = new byte[1000];
            RANDOM.nextBytes(usermetadata1);
            int blobContentSize = 2000;
            byte[] data1 = new byte[blobContentSize];
            RANDOM.nextBytes(data1);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data1 = byteBufferBlob.array();
                blobContentSize = data1.length;
            }
            ByteBufferInputStream stream1 = new ByteBufferInputStream(ByteBuffer.wrap(data1));
            MessageFormatInputStream messageFormatStream1 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key1, ByteBuffer.wrap(encryptionKey1), prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key1, prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType);
            MessageInfo msgInfo1 = new MessageInfo(key1, messageFormatStream1.getSize(), accountId, containerId, prop1.getCreationTimeInMs());
            // create message stream for blob 2 and mark it as deleted
            StoreKey key2 = new MockId("id2");
            accountId = Utils.getRandomShort(RANDOM);
            containerId = Utils.getRandomShort(RANDOM);
            long deletionTimeMs = SystemTime.getInstance().milliseconds() + RANDOM.nextInt();
            MessageFormatInputStream messageFormatStream2 = new DeleteMessageFormatInputStream(key2, accountId, containerId, deletionTimeMs);
            MessageInfo msgInfo2 = new MessageInfo(key2, messageFormatStream2.getSize(), accountId, containerId, deletionTimeMs);
            // create message stream for blob 3
            StoreKey key3 = new MockId("id3");
            accountId = Utils.getRandomShort(RANDOM);
            containerId = Utils.getRandomShort(RANDOM);
            BlobProperties prop3 = new BlobProperties(10, "servid3", accountId, containerId, false);
            byte[] encryptionKey3 = new byte[100];
            RANDOM.nextBytes(encryptionKey3);
            byte[] usermetadata3 = new byte[1000];
            RANDOM.nextBytes(usermetadata3);
            blobContentSize = 2000;
            byte[] data3 = new byte[blobContentSize];
            RANDOM.nextBytes(data3);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data3 = byteBufferBlob.array();
                blobContentSize = data3.length;
            }
            ByteBufferInputStream stream3 = new ByteBufferInputStream(ByteBuffer.wrap(data3));
            MessageFormatInputStream messageFormatStream3 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key3, ByteBuffer.wrap(encryptionKey3), prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key3, prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType);
            MessageInfo msgInfo3 = new MessageInfo(key3, messageFormatStream3.getSize(), accountId, containerId, prop3.getCreationTimeInMs());
            // create input stream for all blob messages together
            byte[] totalMessageContent = new byte[(int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize() + (int) messageFormatStream3.getSize()];
            messageFormatStream1.read(totalMessageContent, 0, (int) messageFormatStream1.getSize());
            messageFormatStream2.read(totalMessageContent, (int) messageFormatStream1.getSize(), (int) messageFormatStream2.getSize());
            messageFormatStream3.read(totalMessageContent, (int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize(), (int) messageFormatStream3.getSize());
            InputStream inputStream = new ByteBufferInputStream(ByteBuffer.wrap(totalMessageContent));
            List<MessageInfo> msgInfoList = new ArrayList<MessageInfo>();
            msgInfoList.add(msgInfo1);
            msgInfoList.add(msgInfo2);
            msgInfoList.add(msgInfo3);
            new MessageSievingInputStream(inputStream, msgInfoList, transformers, new MetricRegistry());
            if (!options.isEmpty()) {
                Assert.fail("IOException should have been thrown due to delete record ");
            }
        }
    } catch (IOException e) {
        if (options.isEmpty()) {
            Assert.fail("No exceptions should have occurred");
        }
    }
    headerVersionToUse = Message_Header_Version_V1;
}

Example 12 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatTtlUpdateRecordTest.

/**
 * Tests for {@link TtlUpdateMessageFormatInputStream} in different versions.
 */
@Test
public void messageFormatTtlUpdateRecordTest() throws IOException, MessageFormatException {
    StoreKey key = new MockId("id1");
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long ttlUpdateTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
    long updatedExpiryMs = ttlUpdateTimeMs + TestUtils.RANDOM.nextInt();
    short lifeVersion = 1;
    short oldMessageFormatHeaderVersion = MessageFormatRecord.headerVersionToUse;
    for (short messageFormatHeaderVersion : new short[] { MessageFormatRecord.Message_Header_Version_V2, MessageFormatRecord.Message_Header_Version_V3 }) {
        MessageFormatRecord.headerVersionToUse = messageFormatHeaderVersion;
        MessageFormatInputStream messageFormatStream = new TtlUpdateMessageFormatInputStream(key, accountId, containerId, updatedExpiryMs, ttlUpdateTimeMs, lifeVersion);
        long ttlUpdateRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.TTL_UPDATE);
        int headerSize = MessageFormatRecord.getHeaderSizeForVersion(MessageFormatRecord.headerVersionToUse);
        Assert.assertEquals(headerSize + ttlUpdateRecordSize + key.sizeInBytes(), messageFormatStream.getSize());
        checkTtlUpdateMessage(messageFormatStream, ttlUpdateRecordSize, key, accountId, containerId, updatedExpiryMs, ttlUpdateTimeMs, lifeVersion);
    }
    MessageFormatRecord.headerVersionToUse = oldMessageFormatHeaderVersion;
}

Example 13 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatPutNoArgReadTest.

/**
 * Test calling the no-arg read method
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void messageFormatPutNoArgReadTest() throws Exception, MessageFormatException {
    StoreKey key = new MockId("id1");
    StoreKeyFactory keyFactory = new MockIdFactory();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties prop = new BlobProperties(10, "servid", accountId, containerId, false);
    byte[] encryptionKey = new byte[100];
    new Random().nextBytes(encryptionKey);
    byte[] usermetadata = new byte[1000];
    new Random().nextBytes(usermetadata);
    int blobContentSize = 2000;
    byte[] data = new byte[blobContentSize];
    new Random().nextBytes(data);
    ByteBufferInputStream stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
    MessageFormatInputStream messageFormatStream = new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, BlobType.DataBlob);
    TestUtils.validateInputStreamContract(messageFormatStream);
    TestUtils.readInputStreamAndValidateSize(messageFormatStream, messageFormatStream.getSize());
}

Example 14 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatDeleteRecordTest.

/**
 * Tests for {@link DeleteMessageFormatInputStream} in different versions.
 */
@Test
public void messageFormatDeleteRecordTest() throws IOException, MessageFormatException {
    short[] versions = { MessageFormatRecord.Update_Version_V1, MessageFormatRecord.Update_Version_V2, MessageFormatRecord.Update_Version_V3 };
    for (short version : versions) {
        StoreKey key = new MockId("id1");
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        long deletionTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
        short lifeVersion = 1;
        MessageFormatInputStream messageFormatStream;
        short messageHeaderVersionInUse;
        int deleteRecordSize;
        if (version == MessageFormatRecord.Update_Version_V1) {
            messageFormatStream = new DeleteMessageFormatV1InputStream(key, accountId, containerId, deletionTimeMs);
            deleteRecordSize = MessageFormatRecord.Update_Format_V1.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.Message_Header_Version_V1;
            // reset account, container ids and time
            accountId = Account.UNKNOWN_ACCOUNT_ID;
            containerId = Container.UNKNOWN_CONTAINER_ID;
            deletionTimeMs = Utils.Infinite_Time;
        } else if (version == MessageFormatRecord.Update_Version_V2) {
            messageFormatStream = new DeleteMessageFormatV2InputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V2.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        } else {
            messageFormatStream = new DeleteMessageFormatInputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.DELETE);
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        }
        int headerSize = MessageFormatRecord.getHeaderSizeForVersion(messageHeaderVersionInUse);
        Assert.assertEquals("Unexpected size for version " + version, headerSize + deleteRecordSize + key.sizeInBytes(), messageFormatStream.getSize());
        // check header
        byte[] headerOutput = new byte[headerSize];
        messageFormatStream.read(headerOutput);
        ByteBuffer headerBuf = ByteBuffer.wrap(headerOutput);
        Assert.assertEquals(messageHeaderVersionInUse, headerBuf.getShort());
        if (messageHeaderVersionInUse == MessageFormatRecord.Message_Header_Version_V3) {
            Assert.assertEquals(lifeVersion, headerBuf.getShort());
        }
        Assert.assertEquals(deleteRecordSize, headerBuf.getLong());
        // read encryption key relative offset
        if (messageHeaderVersionInUse >= MessageFormatRecord.Message_Header_Version_V2) {
            Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        }
        // blob properties relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // delete record relative offset. This is the only relative offset with a valid value.
        Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
        // user metadata relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // blob relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        Crc32 crc = new Crc32();
        crc.update(headerOutput, 0, headerSize - MessageFormatRecord.Crc_Size);
        Assert.assertEquals(crc.getValue(), headerBuf.getLong());
        // verify handle
        byte[] handleOutput = new byte[key.sizeInBytes()];
        messageFormatStream.read(handleOutput);
        Assert.assertArrayEquals(handleOutput, key.toBytes());
        // check delete record
        UpdateRecord updateRecord = MessageFormatRecord.deserializeUpdateRecord(messageFormatStream);
        Assert.assertEquals("Type of update record not DELETE", SubRecord.Type.DELETE, updateRecord.getType());
        Assert.assertNotNull("DeleteSubRecord should not be null", updateRecord.getDeleteSubRecord());
        Assert.assertEquals("AccountId mismatch", accountId, updateRecord.getAccountId());
        Assert.assertEquals("ContainerId mismatch", containerId, updateRecord.getContainerId());
        Assert.assertEquals("DeletionTime mismatch", deletionTimeMs, updateRecord.getUpdateTimeInMs());
    }
}

Example 15 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatUndeleteUpdateRecordTest.

/**
 * Test for {@link UndeleteMessageFormatInputStream}.
 * @throws Exception
 */
@Test
public void messageFormatUndeleteUpdateRecordTest() throws Exception {
    MessageFormatRecord.headerVersionToUse = MessageFormatRecord.Message_Header_Version_V3;
    StoreKey key = new MockId("id1");
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long updateTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
    short lifeVersion = Utils.getRandomShort(TestUtils.RANDOM);
    MessageFormatInputStream messageFormatInputStream = new UndeleteMessageFormatInputStream(key, accountId, containerId, updateTimeMs, lifeVersion);
    long undeleteRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.UNDELETE);
    // Undelete record's version will start at least from V3.
    int headerSize = MessageFormatRecord.getHeaderSizeForVersion(MessageFormatRecord.Message_Header_Version_V3);
    Assert.assertEquals(headerSize + undeleteRecordSize + key.sizeInBytes(), messageFormatInputStream.getSize());
    checkUndeleteMessage(messageFormatInputStream, undeleteRecordSize, key, accountId, containerId, updateTimeMs, lifeVersion);
}