Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 6 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageSievingInputStreamTest method testDeletedBlobs.

private void testDeletedBlobs(short blobVersion, BlobType blobType) throws IOException, MessageFormatException {
    // MessageSievingInputStream contains put records for 2 valid blobs and 1 deleted blob
    // id1(put record for valid blob), id2(delete record) and id3(put record for valid blob)
    ArrayList<Short> versions = new ArrayList<>();
    versions.add(Message_Header_Version_V1);
    if (blobVersion != Blob_Version_V1) {
        versions.add(Message_Header_Version_V2);
    }
    try {
        for (short version : versions) {
            headerVersionToUse = version;
            // create message stream for blob 1
            StoreKey key1 = new MockId("id1");
            short accountId = Utils.getRandomShort(TestUtils.RANDOM);
            short containerId = Utils.getRandomShort(TestUtils.RANDOM);
            BlobProperties prop1 = new BlobProperties(10, "servid1", accountId, containerId, false);
            byte[] encryptionKey1 = new byte[100];
            TestUtils.RANDOM.nextBytes(encryptionKey1);
            byte[] usermetadata1 = new byte[1000];
            TestUtils.RANDOM.nextBytes(usermetadata1);
            int blobContentSize = 2000;
            byte[] data1 = new byte[blobContentSize];
            TestUtils.RANDOM.nextBytes(data1);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data1 = byteBufferBlob.array();
                blobContentSize = data1.length;
            }
            ByteBufferInputStream stream1 = new ByteBufferInputStream(ByteBuffer.wrap(data1));
            MessageFormatInputStream messageFormatStream1 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key1, ByteBuffer.wrap(encryptionKey1), prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key1, prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType);
            MessageInfo msgInfo1 = new MessageInfo(key1, messageFormatStream1.getSize(), accountId, containerId, prop1.getCreationTimeInMs());
            // create message stream for blob 2 and mark it as deleted
            StoreKey key2 = new MockId("id2");
            accountId = Utils.getRandomShort(TestUtils.RANDOM);
            containerId = Utils.getRandomShort(TestUtils.RANDOM);
            long deletionTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
            MessageFormatInputStream messageFormatStream2 = new DeleteMessageFormatInputStream(key2, accountId, containerId, deletionTimeMs);
            MessageInfo msgInfo2 = new MessageInfo(key2, messageFormatStream2.getSize(), accountId, containerId, deletionTimeMs);
            // create message stream for blob 3
            StoreKey key3 = new MockId("id3");
            accountId = Utils.getRandomShort(TestUtils.RANDOM);
            containerId = Utils.getRandomShort(TestUtils.RANDOM);
            BlobProperties prop3 = new BlobProperties(10, "servid3", accountId, containerId, false);
            byte[] encryptionKey3 = new byte[100];
            TestUtils.RANDOM.nextBytes(encryptionKey3);
            byte[] usermetadata3 = new byte[1000];
            TestUtils.RANDOM.nextBytes(usermetadata3);
            blobContentSize = 2000;
            byte[] data3 = new byte[blobContentSize];
            TestUtils.RANDOM.nextBytes(data3);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data3 = byteBufferBlob.array();
                blobContentSize = data3.length;
            }
            ByteBufferInputStream stream3 = new ByteBufferInputStream(ByteBuffer.wrap(data3));
            MessageFormatInputStream messageFormatStream3 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key3, ByteBuffer.wrap(encryptionKey3), prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key3, prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType);
            MessageInfo msgInfo3 = new MessageInfo(key3, messageFormatStream3.getSize(), accountId, containerId, prop3.getCreationTimeInMs());
            // create input stream for all blob messages together
            byte[] totalMessageContent = new byte[(int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize() + (int) messageFormatStream3.getSize()];
            messageFormatStream1.read(totalMessageContent, 0, (int) messageFormatStream1.getSize());
            messageFormatStream2.read(totalMessageContent, (int) messageFormatStream1.getSize(), (int) messageFormatStream2.getSize());
            messageFormatStream3.read(totalMessageContent, (int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize(), (int) messageFormatStream3.getSize());
            InputStream inputStream = new ByteBufferInputStream(ByteBuffer.wrap(totalMessageContent));
            List<MessageInfo> msgInfoList = new ArrayList<MessageInfo>();
            msgInfoList.add(msgInfo1);
            msgInfoList.add(msgInfo2);
            msgInfoList.add(msgInfo3);
            MessageSievingInputStream validMessageDetectionInputStream = new MessageSievingInputStream(inputStream, msgInfoList, new MockIdFactory(), new MetricRegistry());
            Assert.fail("IllegalStateException should have been thrown due to delete record ");
        }
    } catch (IllegalStateException e) {
        Assert.assertTrue("IllegalStateException thrown as expected ", true);
    }
    headerVersionToUse = Message_Header_Version_V1;
}

Example 7 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreDeletes.

/**
 * Test the CloudBlobStore delete method.
 */
@Test
public void testStoreDeletes() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    int count = 10;
    long now = System.currentTimeMillis();
    Map<BlobId, MessageInfo> messageInfoMap = new HashMap<>();
    for (int j = 0; j < count; j++) {
        BlobId blobId = getUniqueId(refAccountId, refContainerId, true, partitionId);
        messageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, initLifeVersion()));
    }
    store.delete(new ArrayList<>(messageInfoMap.values()));
    verify(dest, times(count)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count, 0);
    } else {
        verifyCacheHits(0, 0);
    }
    // If not isVcr, deletion should fail
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    int expectedCount = isVcr ? count : count * 2;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 2, count);
    } else {
        verifyCacheHits(0, 0);
    }
    // Call again with a smaller life version. If isVcr, should hit the cache again.
    Map<BlobId, MessageInfo> newMessageInfoMap = new HashMap<>();
    for (BlobId blobId : messageInfoMap.keySet()) {
        newMessageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, initLifeVersion()));
    }
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    expectedCount = isVcr ? count : count * 3;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 3, count * 2);
    } else {
        verifyCacheHits(0, 0);
    }
    // Call again with a larger life version. Should not hit cache again.
    for (BlobId blobId : messageInfoMap.keySet()) {
        newMessageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 2));
    }
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    expectedCount = isVcr ? count : count * 4;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 4, count * 3);
    } else {
        verifyCacheHits(0, 0);
    }
    // Try to upload a set of blobs containing duplicates
    List<MessageInfo> messageInfoList = new ArrayList<>(messageInfoMap.values());
    messageInfoList.add(messageInfoList.get(messageInfoMap.values().size() - 1));
    try {
        store.delete(messageInfoList);
    } catch (IllegalArgumentException iaex) {
    }
    if (isVcr) {
        verifyCacheHits(count * 4, count * 3);
    } else {
        verifyCacheHits(0, 0);
    }
}

Example 8 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreTtlUpdates.

/**
 * Test the CloudBlobStore updateTtl method.
 */
@Test
public void testStoreTtlUpdates() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 10;
    for (int j = 0; j < count; j++) {
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, -1, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    }
    store.updateTtl(messageWriteSet.getMessageSetInfo());
    verify(dest, times(count)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    verifyCacheHits(count, 0);
    // Call second time, If isVcr, should be cached causing updates to be skipped.
    store.updateTtl(messageWriteSet.getMessageSetInfo());
    int expectedCount = isVcr ? count : count * 2;
    verify(dest, times(expectedCount)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    verifyCacheHits(count * 2, count);
    // test that if a blob is deleted and then undeleted, the ttlupdate status is preserved in cache.
    MessageInfo messageInfo = messageWriteSet.getMessageSetInfo().get(0);
    store.delete(Collections.singletonList(messageInfo));
    verify(dest, times(1)).deleteBlob(any(BlobId.class), anyLong(), anyShort(), any(CloudUpdateValidator.class));
    store.undelete(messageInfo);
    verify(dest, times(1)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    store.updateTtl(Collections.singletonList(messageInfo));
    expectedCount = isVcr ? expectedCount : expectedCount + 1;
    verify(dest, times(expectedCount)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits((count * 2) + 3, count + 1);
    } else {
        // delete and undelete should not cause cache lookup for frontend.
        verifyCacheHits((count * 2) + 1, count + 1);
    }
    // test that ttl update with non infinite expiration time fails
    messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, System.currentTimeMillis() + 20000, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    try {
        store.updateTtl(messageWriteSet.getMessageSetInfo());
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.Update_Not_Allowed);
    }
}

Example 9 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testStorePuts.

/**
 * Test the CloudBlobStore put method with specified encryption.
 * @param requireEncryption true for encrypted puts. false otherwise.
 */
private void testStorePuts(boolean requireEncryption) throws Exception {
    setupCloudStore(true, requireEncryption, defaultCacheLimit, true);
    // Put blobs with and without expiration and encryption
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 5;
    int expectedUploads = 0;
    long expectedBytesUploaded = 0;
    int expectedEncryptions = 0;
    for (int j = 0; j < count; j++) {
        long size = Math.abs(random.nextLong()) % 10000;
        // Permanent and encrypted, should be uploaded and not reencrypted
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
        expectedUploads++;
        expectedBytesUploaded += size;
        // Permanent and unencrypted
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr);
        expectedUploads++;
        expectedBytesUploaded += size;
        if (requireEncryption) {
            expectedEncryptions++;
        }
    }
    store.put(messageWriteSet);
    LatchBasedInMemoryCloudDestination inMemoryDest = (LatchBasedInMemoryCloudDestination) dest;
    assertEquals("Unexpected blobs count", expectedUploads, inMemoryDest.getBlobsUploaded());
    assertEquals("Unexpected byte count", expectedBytesUploaded, inMemoryDest.getBytesUploaded());
    assertEquals("Unexpected encryption count", expectedEncryptions, vcrMetrics.blobEncryptionCount.getCount());
    verifyCacheHits(expectedUploads, 0);
    // Try to put the same blobs again (e.g. from another replica).
    // If isVcr is true, they should already be cached.
    messageWriteSet.resetBuffers();
    for (MessageInfo messageInfo : messageWriteSet.getMessageSetInfo()) {
        try {
            MockMessageWriteSet mockMessageWriteSet = new MockMessageWriteSet();
            CloudTestUtil.addBlobToMessageSet(mockMessageWriteSet, (BlobId) messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.getExpirationTimeInMs(), operationTime, isVcr);
            store.put(mockMessageWriteSet);
            fail("Uploading already uploaded blob shoudl throw error");
        } catch (StoreException ex) {
            assertEquals(ex.getErrorCode(), StoreErrorCodes.Already_Exist);
        }
    }
    int expectedSkips = isVcr ? expectedUploads : 0;
    assertEquals("Unexpected blobs count", expectedUploads, inMemoryDest.getBlobsUploaded());
    assertEquals("Unexpected byte count", expectedBytesUploaded, inMemoryDest.getBytesUploaded());
    assertEquals("Unexpected skipped count", expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    // Try to upload a set of blobs containing duplicates
    MessageInfo duplicateMessageInfo = messageWriteSet.getMessageSetInfo().get(messageWriteSet.getMessageSetInfo().size() - 1);
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, (BlobId) duplicateMessageInfo.getStoreKey(), duplicateMessageInfo.getSize(), duplicateMessageInfo.getExpirationTimeInMs(), operationTime, isVcr);
    try {
        store.put(messageWriteSet);
    } catch (IllegalArgumentException iaex) {
    }
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    // Verify that a blob marked as deleted is not uploaded.
    MockMessageWriteSet deletedMessageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(deletedMessageWriteSet, 100, Utils.Infinite_Time, refAccountId, refContainerId, true, true, partitionId, operationTime, isVcr);
    store.put(deletedMessageWriteSet);
    expectedSkips++;
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    assertEquals(expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
    // Verify that a blob that is expiring soon is not uploaded for vcr but is uploaded for frontend.
    messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, 100, operationTime + cloudConfig.vcrMinTtlDays - 1, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    store.put(messageWriteSet);
    int expectedLookups = (2 * expectedUploads) + 1;
    if (isVcr) {
        expectedSkips++;
    } else {
        expectedUploads++;
    }
    verifyCacheHits(expectedLookups, expectedUploads);
    assertEquals(expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
}

Example 10 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testGetForDeletedBlobWithIncludeExpiredOption.

/**
 * Test cloud store get with a list of blobs such that atleast one of them is expired, and {@code StoreGetOptions.Store_Include_Expired} is set in getoptions
 * @param blobIds list of blobids to get
 * @param expiredBlobId expired blob id
 * @throws Exception
 */
private void testGetForDeletedBlobWithIncludeExpiredOption(List<BlobId> blobIds, BlobId expiredBlobId) throws Exception {
    StoreInfo storeInfo = store.get(blobIds, EnumSet.of(StoreGetOptions.Store_Include_Expired));
    for (MessageInfo msgInfo : storeInfo.getMessageReadSetInfo()) {
        if (msgInfo.getStoreKey().equals(expiredBlobId)) {
            return;
        }
    }
    fail("get should be successful for a expired blob with Store_Include_Expired set in get options");
}