Examples with MockMessageWriteSet com.github.ambry.store.MockMessageWriteSet used on opensource projects

Example 1 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testExceptionalDest.

/**
 * Test verifying exception handling behavior.
 */
@Test
public void testExceptionalDest() throws Exception {
    CloudDestination exDest = mock(CloudDestination.class);
    when(exDest.uploadBlob(any(BlobId.class), anyLong(), any(), any(InputStream.class))).thenThrow(new CloudStorageException("ouch"));
    when(exDest.deleteBlob(any(BlobId.class), anyLong(), anyShort(), any(CloudUpdateValidator.class))).thenThrow(new CloudStorageException("ouch"));
    when(exDest.getBlobMetadata(anyList())).thenThrow(new CloudStorageException("ouch"));
    Properties props = new Properties();
    setBasicProperties(props);
    props.setProperty(CloudConfig.VCR_REQUIRE_ENCRYPTION, "false");
    props.setProperty(CloudConfig.CLOUD_BLOB_CRYPTO_AGENT_FACTORY_CLASS, TestCloudBlobCryptoAgentFactory.class.getName());
    vcrMetrics = new VcrMetrics(new MetricRegistry());
    CloudBlobStore exStore = new CloudBlobStore(new VerifiableProperties(props), partitionId, exDest, clusterMap, vcrMetrics);
    exStore.start();
    List<StoreKey> keys = Collections.singletonList(getUniqueId(refAccountId, refContainerId, false, partitionId));
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, 10, Utils.Infinite_Time, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    try {
        exStore.put(messageWriteSet);
        fail("Store put should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.IOError, e.getErrorCode());
    }
    try {
        exStore.delete(messageWriteSet.getMessageSetInfo());
        fail("Store delete should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.IOError, e.getErrorCode());
    }
    try {
        exStore.findMissingKeys(keys);
        fail("Store findMissingKeys should have failed.");
    } catch (StoreException e) {
        assertEquals(StoreErrorCodes.IOError, e.getErrorCode());
    }
}

Example 2 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testExceptionRetry.

/* Test retry behavior */
@Test
public void testExceptionRetry() throws Exception {
    CloudDestination exDest = mock(CloudDestination.class);
    long retryDelay = 5;
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    BlobId blobId = CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr);
    List<StoreKey> keys = Collections.singletonList(blobId);
    CloudBlobMetadata metadata = new CloudBlobMetadata(blobId, operationTime, Utils.Infinite_Time, 1024, null);
    CloudStorageException retryableException = new CloudStorageException("Server unavailable", null, 500, true, retryDelay);
    when(exDest.uploadBlob(any(BlobId.class), anyLong(), any(), any(InputStream.class))).thenAnswer((invocation -> {
        consumeStream(invocation);
        throw retryableException;
    })).thenAnswer(invocation -> {
        consumeStream(invocation);
        return true;
    });
    when(exDest.deleteBlob(any(BlobId.class), anyLong(), anyShort(), any(CloudUpdateValidator.class))).thenThrow(retryableException).thenReturn(true);
    when(exDest.updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class))).thenThrow(retryableException).thenReturn((short) 1);
    when(exDest.getBlobMetadata(anyList())).thenThrow(retryableException).thenReturn(Collections.singletonMap(metadata.getId(), metadata));
    doThrow(retryableException).doNothing().when(exDest).downloadBlob(any(BlobId.class), any());
    Properties props = new Properties();
    setBasicProperties(props);
    props.setProperty(CloudConfig.VCR_REQUIRE_ENCRYPTION, "false");
    props.setProperty(CloudConfig.CLOUD_BLOB_CRYPTO_AGENT_FACTORY_CLASS, TestCloudBlobCryptoAgentFactory.class.getName());
    vcrMetrics = new VcrMetrics(new MetricRegistry());
    CloudBlobStore exStore = spy(new CloudBlobStore(new VerifiableProperties(props), partitionId, exDest, clusterMap, vcrMetrics));
    exStore.start();
    // Run all operations, they should be retried and succeed second time.
    int expectedCacheLookups = 0, expectedCacheRemoves = 0, expectedRetries = 0;
    // PUT
    exStore.put(messageWriteSet);
    expectedRetries++;
    expectedCacheLookups++;
    // UPDATE_TTL
    exStore.updateTtl(messageWriteSet.getMessageSetInfo());
    expectedRetries++;
    expectedCacheRemoves++;
    expectedCacheLookups += 2;
    // DELETE
    exStore.delete(messageWriteSet.getMessageSetInfo());
    expectedRetries++;
    expectedCacheRemoves++;
    if (isVcr) {
        // no cache lookup in case of delete for frontend.
        expectedCacheLookups += 2;
    }
    // GET
    exStore.get(keys, EnumSet.noneOf(StoreGetOptions.class));
    expectedRetries++;
    exStore.downloadBlob(metadata, blobId, new ByteArrayOutputStream());
    expectedRetries++;
    // Expect retries for all ops, cache lookups for the first three
    assertEquals("Unexpected retry count", expectedRetries, vcrMetrics.retryCount.getCount());
    assertEquals("Unexpected wait time", expectedRetries * retryDelay, vcrMetrics.retryWaitTimeMsec.getCount());
    verifyCacheHits(expectedCacheLookups, 0);
    verify(exStore, times(expectedCacheRemoves)).removeFromCache(eq(blobId.getID()));
    // Rerun the first three, should all skip due to cache hit
    messageWriteSet.resetBuffers();
    int expectedCacheHits = 0;
    try {
        exStore.put(messageWriteSet);
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.Already_Exist);
    }
    expectedCacheHits++;
    exStore.addToCache(blobId.getID(), (short) 0, CloudBlobStore.BlobState.TTL_UPDATED);
    exStore.updateTtl(messageWriteSet.getMessageSetInfo());
    expectedCacheHits++;
    exStore.addToCache(blobId.getID(), (short) 0, CloudBlobStore.BlobState.DELETED);
    try {
        exStore.delete(messageWriteSet.getMessageSetInfo());
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    if (isVcr) {
        expectedCacheHits++;
    }
    verifyCacheHits(expectedCacheLookups + expectedCacheHits, expectedCacheHits);
}

Example 3 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreTtlUpdates.

/**
 * Test the CloudBlobStore updateTtl method.
 */
@Test
public void testStoreTtlUpdates() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 10;
    for (int j = 0; j < count; j++) {
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, -1, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    }
    store.updateTtl(messageWriteSet.getMessageSetInfo());
    verify(dest, times(count)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    verifyCacheHits(count, 0);
    // Call second time, If isVcr, should be cached causing updates to be skipped.
    store.updateTtl(messageWriteSet.getMessageSetInfo());
    int expectedCount = isVcr ? count : count * 2;
    verify(dest, times(expectedCount)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    verifyCacheHits(count * 2, count);
    // test that if a blob is deleted and then undeleted, the ttlupdate status is preserved in cache.
    MessageInfo messageInfo = messageWriteSet.getMessageSetInfo().get(0);
    store.delete(Collections.singletonList(messageInfo));
    verify(dest, times(1)).deleteBlob(any(BlobId.class), anyLong(), anyShort(), any(CloudUpdateValidator.class));
    store.undelete(messageInfo);
    verify(dest, times(1)).undeleteBlob(any(BlobId.class), anyShort(), any(CloudUpdateValidator.class));
    store.updateTtl(Collections.singletonList(messageInfo));
    expectedCount = isVcr ? expectedCount : expectedCount + 1;
    verify(dest, times(expectedCount)).updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits((count * 2) + 3, count + 1);
    } else {
        // delete and undelete should not cause cache lookup for frontend.
        verifyCacheHits((count * 2) + 1, count + 1);
    }
    // test that ttl update with non infinite expiration time fails
    messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, System.currentTimeMillis() + 20000, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    try {
        store.updateTtl(messageWriteSet.getMessageSetInfo());
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.Update_Not_Allowed);
    }
}

Example 4 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreTest method testStorePuts.

/**
 * Test the CloudBlobStore put method with specified encryption.
 * @param requireEncryption true for encrypted puts. false otherwise.
 */
private void testStorePuts(boolean requireEncryption) throws Exception {
    setupCloudStore(true, requireEncryption, defaultCacheLimit, true);
    // Put blobs with and without expiration and encryption
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 5;
    int expectedUploads = 0;
    long expectedBytesUploaded = 0;
    int expectedEncryptions = 0;
    for (int j = 0; j < count; j++) {
        long size = Math.abs(random.nextLong()) % 10000;
        // Permanent and encrypted, should be uploaded and not reencrypted
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
        expectedUploads++;
        expectedBytesUploaded += size;
        // Permanent and unencrypted
        CloudTestUtil.addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr);
        expectedUploads++;
        expectedBytesUploaded += size;
        if (requireEncryption) {
            expectedEncryptions++;
        }
    }
    store.put(messageWriteSet);
    LatchBasedInMemoryCloudDestination inMemoryDest = (LatchBasedInMemoryCloudDestination) dest;
    assertEquals("Unexpected blobs count", expectedUploads, inMemoryDest.getBlobsUploaded());
    assertEquals("Unexpected byte count", expectedBytesUploaded, inMemoryDest.getBytesUploaded());
    assertEquals("Unexpected encryption count", expectedEncryptions, vcrMetrics.blobEncryptionCount.getCount());
    verifyCacheHits(expectedUploads, 0);
    // Try to put the same blobs again (e.g. from another replica).
    // If isVcr is true, they should already be cached.
    messageWriteSet.resetBuffers();
    for (MessageInfo messageInfo : messageWriteSet.getMessageSetInfo()) {
        try {
            MockMessageWriteSet mockMessageWriteSet = new MockMessageWriteSet();
            CloudTestUtil.addBlobToMessageSet(mockMessageWriteSet, (BlobId) messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.getExpirationTimeInMs(), operationTime, isVcr);
            store.put(mockMessageWriteSet);
            fail("Uploading already uploaded blob shoudl throw error");
        } catch (StoreException ex) {
            assertEquals(ex.getErrorCode(), StoreErrorCodes.Already_Exist);
        }
    }
    int expectedSkips = isVcr ? expectedUploads : 0;
    assertEquals("Unexpected blobs count", expectedUploads, inMemoryDest.getBlobsUploaded());
    assertEquals("Unexpected byte count", expectedBytesUploaded, inMemoryDest.getBytesUploaded());
    assertEquals("Unexpected skipped count", expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    // Try to upload a set of blobs containing duplicates
    MessageInfo duplicateMessageInfo = messageWriteSet.getMessageSetInfo().get(messageWriteSet.getMessageSetInfo().size() - 1);
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, (BlobId) duplicateMessageInfo.getStoreKey(), duplicateMessageInfo.getSize(), duplicateMessageInfo.getExpirationTimeInMs(), operationTime, isVcr);
    try {
        store.put(messageWriteSet);
    } catch (IllegalArgumentException iaex) {
    }
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    // Verify that a blob marked as deleted is not uploaded.
    MockMessageWriteSet deletedMessageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(deletedMessageWriteSet, 100, Utils.Infinite_Time, refAccountId, refContainerId, true, true, partitionId, operationTime, isVcr);
    store.put(deletedMessageWriteSet);
    expectedSkips++;
    verifyCacheHits(2 * expectedUploads, expectedUploads);
    assertEquals(expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
    // Verify that a blob that is expiring soon is not uploaded for vcr but is uploaded for frontend.
    messageWriteSet = new MockMessageWriteSet();
    CloudTestUtil.addBlobToMessageSet(messageWriteSet, 100, operationTime + cloudConfig.vcrMinTtlDays - 1, refAccountId, refContainerId, true, false, partitionId, operationTime, isVcr);
    store.put(messageWriteSet);
    int expectedLookups = (2 * expectedUploads) + 1;
    if (isVcr) {
        expectedSkips++;
    } else {
        expectedUploads++;
    }
    verifyCacheHits(expectedLookups, expectedUploads);
    assertEquals(expectedSkips, vcrMetrics.blobUploadSkippedCount.getCount());
}

Example 5 with MockMessageWriteSet

use of com.github.ambry.store.MockMessageWriteSet in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testPut.

/**
 * Test {@link CloudBlobStore#put} method.
 */
@Test
public void testPut() throws StoreException {
    // Put blobs with and without expiration and encryption
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    int count = 5;
    int expectedUploads = 0;
    int expectedEncryptions = 0;
    for (int j = 0; j < count; j++) {
        long size = Math.abs(random.nextLong()) % 10000;
        // Permanent and encrypted, should be uploaded and not reencrypted
        addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, accountId, containerId, true, false, partitionId, operationTime, isVcr);
        expectedUploads++;
        // Permanent and unencrypted
        addBlobToMessageSet(messageWriteSet, size, Utils.Infinite_Time, accountId, containerId, false, false, partitionId, operationTime, isVcr);
        expectedUploads++;
    }
    cloudBlobStore.put(messageWriteSet);
    assertEquals("Unexpected blobs count", expectedUploads, azureMetrics.blobUploadSuccessCount.getCount());
    assertEquals("Unexpected encryption count", expectedEncryptions, vcrMetrics.blobEncryptionCount.getCount());
}