Examples with MockMessageWriteSet com.github.ambry.store.MockMessageWriteSet used on opensource projects

Example 11 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreGets.

/**
 * Test cloud store get method with the given encryption requirement.
 * @throws Exception
 */
private void testStoreGets(boolean requireEncryption) throws Exception {
    setupCloudStore(true, requireEncryption, defaultCacheLimit, true);
    // Put blobs with and without expiration and encryption
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 5;
    List<BlobId> blobIds = new ArrayList<>(count);
    Map<BlobId, ByteBuffer> blobIdToUploadedDataMap = new HashMap<>(count);
    for (int j = 0; j < count; j++) {
        long size = Math.abs(random.nextLong()) % 10000;
        blobIds.add(CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr));
        ByteBuffer uploadedData = messageWriteSet.getBuffers().get(messageWriteSet.getMessageSetInfo().size() - 1);
        blobIdToUploadedDataMap.put(blobIds.get(j), uploadedData);
    }
    store.put(messageWriteSet);
    for (BlobId blobId : blobIdToUploadedDataMap.keySet()) {
        blobIdToUploadedDataMap.get(blobId).flip();
    }
    // test get for a list of blobs that exist
    testGetForExistingBlobs(blobIds, blobIdToUploadedDataMap);
    // test get for one blob that exists
    List<BlobId> singleIdList = Collections.singletonList(blobIds.get(0));
    Map<BlobId, ByteBuffer> singleBlobIdToUploadedDataMap = new HashMap<>(1);
    singleBlobIdToUploadedDataMap.put(blobIds.get(0), blobIdToUploadedDataMap.get(blobIds.get(0)));
    testGetForExistingBlobs(singleIdList, singleBlobIdToUploadedDataMap);
    // test get for one blob that doesnt exist
    BlobId nonExistentId = getUniqueId(refAccountId, refContainerId, false, partitionId);
    singleIdList = Collections.singletonList(nonExistentId);
    testGetWithAtleastOneNonExistentBlob(singleIdList);
    // test get with one blob in list non-existent
    blobIds.add(nonExistentId);
    testGetWithAtleastOneNonExistentBlob(blobIds);
    blobIds.remove(blobIds.remove(blobIds.size() - 1));
    // test get for deleted blob
    MessageInfo deletedMessageInfo = messageWriteSet.getMessageSetInfo().get(0);
    MockMessageWriteSet deleteMockMessageWriteSet = new MockMessageWriteSet();
    deleteMockMessageWriteSet.add(deletedMessageInfo, null);
    store.delete(deleteMockMessageWriteSet.getMessageSetInfo());
    // test get with a deleted blob in blob list to get, but {@code StoreGetOptions.Store_Include_Deleted} not set in get options
    testGetForDeletedBlobWithoutIncludeDeleteOption(blobIds);
    // test get with a deleted blob in blob list to get, and {@code StoreGetOptions.Store_Include_Deleted} set in get options
    testGetForDeletedBlobWithIncludeDeleteOption(blobIds, (BlobId) deletedMessageInfo.getStoreKey());
    blobIds.remove(0);
    // test get for expired blob
    BlobId expiredBlobId = forceUploadExpiredBlob();
    blobIds.add(expiredBlobId);
    // test get with an expired blob in blob list, but {@code StoreGetOptions.Store_Include_Expired} not set in get options
    testGetForDeletedBlobWithoutIncludeExpiredOption(blobIds);
    // test get with a expired blob in blob list to get, and {@code StoreGetOptions.Store_Include_Expired} set in get options
    testGetForDeletedBlobWithIncludeExpiredOption(blobIds, expiredBlobId);
}

Example 12 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreUndeletes.

/**
 * Test the CloudBlobStore undelete method.
 */
@Test
public void testStoreUndeletes() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    long now = System.currentTimeMillis();
    MessageInfo messageInfo = new MessageInfo(getUniqueId(refAccountId, refContainerId, true, partitionId), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 1);
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 1);
    store.undelete(messageInfo);
    verify(dest, times(1)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(1, 0);
    // Call second time with same life version. If isVcr, should hit cache this time.
    try {
        store.undelete(messageInfo);
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Undeleted);
    }
    int expectedCount = isVcr ? 1 : 2;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), eq((short) 1), any(CloudUpdateValidator.class));
    verifyCacheHits(2, 1);
    // Call again with a smaller life version.
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 0);
    messageInfo = new MessageInfo(messageInfo.getStoreKey(), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 0);
    try {
        store.undelete(messageInfo);
    } catch (StoreException ex) {
        assertEquals(StoreErrorCodes.ID_Undeleted, ex.getErrorCode());
    }
    expectedCount = isVcr ? 1 : 3;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(3, 2);
    // Call again with a higher life version. Should not hit cache this time.
    when(dest.undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class))).thenReturn((short) 2);
    messageInfo = new MessageInfo(messageInfo.getStoreKey(), SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 2);
    store.undelete(messageInfo);
    expectedCount = isVcr ? 2 : 4;
    verify(dest, times(expectedCount)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    verifyCacheHits(4, 2);
    // undelete for a non existent blob.
    setupCloudStore(true, true, defaultCacheLimit, true);
    try {
        store.undelete(messageInfo);
        fail("Undelete for a non existent blob should throw exception");
    } catch (StoreException ex) {
        assertSame(ex.getErrorCode(), StoreErrorCodes.ID_Not_Found);
    }
    // add blob and then undelete should pass
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(SMALL_BLOB_SIZE));
    // currently messageInfo.lifeVersion = 2
    messageWriteSet.add(messageInfo, buffer);
    store.put(messageWriteSet);
    assertEquals(store.undelete(messageInfo), 2);
}

Example 13 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreNotStarted.

/**
 * Test verifying behavior when store not started.
 */
@Test
public void testStoreNotStarted() throws Exception {
    // Create store and don't start it.
    setupCloudStore(false, true, defaultCacheLimit, false);
    List<StoreKey> keys = Collections.singletonList(getUniqueId(refAccountId, refContainerId, false, partitionId));
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, 10, Utils.Infinite_Time, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    try {
        store.put(messageWriteSet);
        fail("Store put should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.Store_Not_Started, e.getErrorCode());
    }
    try {
        store.delete(messageWriteSet.getMessageSetInfo());
        fail("Store delete should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.Store_Not_Started, e.getErrorCode());
    }
    try {
        store.findMissingKeys(keys);
        fail("Store findMissingKeys should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.Store_Not_Started, e.getErrorCode());
    }
}

Example 14 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testDeleteFromVcr.

/**
 * Test {@link CloudBlobStore#delete} method for vcr.
 */
public void testDeleteFromVcr() throws StoreException {
    // First upload a blob with a life version 2
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    addBlobToMessageSet(messageWriteSet, Utils.Infinite_Time, accountId, containerId, partitionId, operationTime, (short) 2);
    cloudBlobStore.put(messageWriteSet);
    // verify that the blob was uploaded with expected metadata.
    StoreInfo storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertEquals("Unexpected delete status", messageWriteSet.getMessageSetInfo().get(0).isDeleted(), storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Now delete with a smaller life version should fail silently without updating the life version.
    MessageInfo messageInfo = messageWriteSet.getMessageSetInfo().get(0);
    MessageInfo deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 1);
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertEquals("Unexpected delete status", messageWriteSet.getMessageSetInfo().get(0).isDeleted(), storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Delete with same life version should pass without changing life version.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), messageInfo.getLifeVersion());
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Deleting a deleted blob with higher life version should update life version.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 3);
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Deleting again with smaller life version should fail with exception.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 1);
    try {
        cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
        fail("Delete should fail with ID_Deleted StoreException");
    } catch (StoreException ex) {
        assertEquals("Unexpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Restart cloud blob store to clear cache. Deleting again with smaller life version should fail silently without updating anything.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
}

Example 15 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testUpdateTtl.

/**
 * Test {@link CloudBlobStore#updateTtl} method.
 */
@Test
public void testUpdateTtl() throws StoreException {
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    StoreConfig storeConfig = new StoreConfig(verifiableProperties);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    long now = System.currentTimeMillis();
    long expirationTimeMs = now;
    if (isVcr) {
        // vcr doesn't upload a blob that is within CloudConfig#vcrMinTtlDays of expiry.
        expirationTimeMs += Math.max(TimeUnit.DAYS.toMillis(cloudConfig.vcrMinTtlDays), TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds));
    } else {
        expirationTimeMs += TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds);
    }
    expirationTimeMs += 100000;
    addBlobToMessageSet(messageWriteSet, expirationTimeMs, accountId, containerId, partitionId, operationTime, (short) -1);
    cloudBlobStore.put(messageWriteSet);
    // verify that the blob was uploaded with expected metadata.
    StoreInfo storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertFalse("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", expirationTimeMs, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update without setting ttl update flag.
    MessageInfo ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update on a updated blob. It should fail silently.
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Clear cache by restarting blob store. Do a ttl update on a updated blob. It should fail silently.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Delete the blob.
    cloudBlobStore.delete(Collections.singletonList(ttlUpdateMessageInfo));
    // ttlupdate of a deleted blob should throw ID_Delete Store Exception for frontend and fail silently for vcr.
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail for frontend.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexcpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    // Clear cache by restarting blob store. ttlupdate of a deleted blob should throw ID_Delete Store Exception.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
}