Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 71 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStore method delete.

@Override
public void delete(List<MessageInfo> infos) throws StoreException {
    checkStarted();
    checkDuplicates(infos);
    try {
        for (MessageInfo msgInfo : infos) {
            BlobId blobId = (BlobId) msgInfo.getStoreKey();
            // If the cache has been updated by another thread, retry may be avoided
            requestAgent.doWithRetries(() -> deleteIfNeeded(blobId, msgInfo.getOperationTimeMs(), msgInfo.getLifeVersion()), "Delete", partitionId.toPathString());
        }
    } catch (CloudStorageException ex) {
        if (ex.getCause() instanceof StoreException) {
            throw (StoreException) ex.getCause();
        }
        StoreErrorCodes errorCode = (ex.getStatusCode() == STATUS_NOT_FOUND) ? StoreErrorCodes.ID_Not_Found : StoreErrorCodes.IOError;
        throw new StoreException(ex, errorCode);
    }
}

Example 72 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreGets.

/**
 * Test cloud store get method with the given encryption requirement.
 * @throws Exception
 */
private void testStoreGets(boolean requireEncryption) throws Exception {
    setupCloudStore(true, requireEncryption, defaultCacheLimit, true);
    // Put blobs with and without expiration and encryption
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 5;
    List<BlobId> blobIds = new ArrayList<>(count);
    Map<BlobId, ByteBuffer> blobIdToUploadedDataMap = new HashMap<>(count);
    for (int j = 0; j < count; j++) {
        long size = Math.abs(random.nextLong()) % 10000;
        blobIds.add(CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr));
        ByteBuffer uploadedData = messageWriteSet.getBuffers().get(messageWriteSet.getMessageSetInfo().size() - 1);
        blobIdToUploadedDataMap.put(blobIds.get(j), uploadedData);
    }
    store.put(messageWriteSet);
    for (BlobId blobId : blobIdToUploadedDataMap.keySet()) {
        blobIdToUploadedDataMap.get(blobId).flip();
    }
    // test get for a list of blobs that exist
    testGetForExistingBlobs(blobIds, blobIdToUploadedDataMap);
    // test get for one blob that exists
    List<BlobId> singleIdList = Collections.singletonList(blobIds.get(0));
    Map<BlobId, ByteBuffer> singleBlobIdToUploadedDataMap = new HashMap<>(1);
    singleBlobIdToUploadedDataMap.put(blobIds.get(0), blobIdToUploadedDataMap.get(blobIds.get(0)));
    testGetForExistingBlobs(singleIdList, singleBlobIdToUploadedDataMap);
    // test get for one blob that doesnt exist
    BlobId nonExistentId = getUniqueId(refAccountId, refContainerId, false, partitionId);
    singleIdList = Collections.singletonList(nonExistentId);
    testGetWithAtleastOneNonExistentBlob(singleIdList);
    // test get with one blob in list non-existent
    blobIds.add(nonExistentId);
    testGetWithAtleastOneNonExistentBlob(blobIds);
    blobIds.remove(blobIds.remove(blobIds.size() - 1));
    // test get for deleted blob
    MessageInfo deletedMessageInfo = messageWriteSet.getMessageSetInfo().get(0);
    MockMessageWriteSet deleteMockMessageWriteSet = new MockMessageWriteSet();
    deleteMockMessageWriteSet.add(deletedMessageInfo, null);
    store.delete(deleteMockMessageWriteSet.getMessageSetInfo());
    // test get with a deleted blob in blob list to get, but {@code StoreGetOptions.Store_Include_Deleted} not set in get options
    testGetForDeletedBlobWithoutIncludeDeleteOption(blobIds);
    // test get with a deleted blob in blob list to get, and {@code StoreGetOptions.Store_Include_Deleted} set in get options
    testGetForDeletedBlobWithIncludeDeleteOption(blobIds, (BlobId) deletedMessageInfo.getStoreKey());
    blobIds.remove(0);
    // test get for expired blob
    BlobId expiredBlobId = forceUploadExpiredBlob();
    blobIds.add(expiredBlobId);
    // test get with an expired blob in blob list, but {@code StoreGetOptions.Store_Include_Expired} not set in get options
    testGetForDeletedBlobWithoutIncludeExpiredOption(blobIds);
    // test get with a expired blob in blob list to get, and {@code StoreGetOptions.Store_Include_Expired} set in get options
    testGetForDeletedBlobWithIncludeExpiredOption(blobIds, expiredBlobId);
}

Example 73 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreUndeletes.

/**
 * Test the CloudBlobStore undelete method.
 */
@Test
public void testStoreUndeletes() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    long now = System.currentTimeMillis();
    MessageInfo messageInfo = new MessageInfo(getUniqueId(refAccountId, refContainerId, true, partitionId), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 1);
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 1);
    store.undelete(messageInfo);
    verify(dest, times(1)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(1, 0);
    // Call second time with same life version. If isVcr, should hit cache this time.
    try {
        store.undelete(messageInfo);
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Undeleted);
    }
    int expectedCount = isVcr ? 1 : 2;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), eq((short) 1), any(CloudUpdateValidator.class));
    verifyCacheHits(2, 1);
    // Call again with a smaller life version.
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 0);
    messageInfo = new MessageInfo(messageInfo.getStoreKey(), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 0);
    try {
        store.undelete(messageInfo);
    } catch (StoreException ex) {
        assertEquals(StoreErrorCodes.ID_Undeleted, ex.getErrorCode());
    }
    expectedCount = isVcr ? 1 : 3;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(3, 2);
    // Call again with a higher life version. Should not hit cache this time.
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 2);
    messageInfo = new MessageInfo(messageInfo.getStoreKey(), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 2);
    store.undelete(messageInfo);
    expectedCount = isVcr ? 2 : 4;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(4, 2);
    // undelete for a non existent blob.
    setupCloudStore(true, true, defaultCacheLimit, true);
    try {
        store.undelete(messageInfo);
        fail("Undelete for a non existent blob should throw exception");
    } catch (StoreException ex) {
        assertSame(ex.getErrorCode(), StoreErrorCodes.ID_Not_Found);
    }
    // add blob and then undelete should pass
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(SMALL_BLOB_SIZE));
    // currently messageInfo.lifeVersion = 2
    messageWriteSet.add(messageInfo, buffer);
    store.put(messageWriteSet);
    assertEquals(store.undelete(messageInfo), 2);
}

Example 74 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testGetForDeletedBlobWithIncludeDeleteOption.

/**
 * Test cloud store get with a list of blobs such that atleast one of them is deleted, and {@code StoreGetOptions.Store_Include_Deleted} is set in getoptions
 * @param blobIds list of blobids to get
 * @param deletedBlobId blobId which is marked as deleted
 * @throws Exception
 */
private void testGetForDeletedBlobWithIncludeDeleteOption(List<BlobId> blobIds, BlobId deletedBlobId) throws Exception {
    StoreInfo storeInfo = store.get(blobIds, EnumSet.of(StoreGetOptions.Store_Include_Deleted));
    for (MessageInfo msgInfo : storeInfo.getMessageReadSetInfo()) {
        if (msgInfo.getStoreKey().equals(deletedBlobId)) {
            return;
        }
    }
    fail("get should be successful for a deleted blob with Store_Include_Deleted set in get options");
}

Example 75 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudTestUtil method addBlobToMessageSet.

/**
 * Utility method to generate a BlobId and byte buffer for a blob with specified properties and add them to the specified MessageWriteSet.
 * @param messageWriteSet the {@link MockMessageWriteSet} in which to store the data.
 * @param expiresAtMs the expiration time.
 * @param accountId the account Id.
 * @param containerId the container Id.
 * @param partitionId the partition id.
 * @param operationTime the operation time.
 * @param lifeVersion the life version.
 */
static void addBlobToMessageSet(MockMessageWriteSet messageWriteSet, long expiresAtMs, short accountId, short containerId, PartitionId partitionId, long operationTime, short lifeVersion) {
    long size = 100;
    BlobId id = getUniqueId(accountId, containerId, false, partitionId);
    long crc = new Random().nextLong();
    MessageInfo info = new MessageInfo(id, size, false, true, false, expiresAtMs, crc, accountId, containerId, operationTime, lifeVersion);
    ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes((int) size));
    messageWriteSet.add(info, buffer);
}