Examples with CloudStorageException com.github.ambry.cloud.CloudStorageException used on opensource projects

Example 6 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureContainerCompactor method saveLatestContainerDeletionTime.

/**
 * Save the deleted container update checkpoint {@code latestContainerDeletionTimestamp} to Azure Blob Store.
 * @param latestContainerDeletionTimestamp timestamp representing deleteTriggerTime upto which deleted containers have been updated in cloud.
 * @throws CloudStorageException in case of any error.
 */
private void saveLatestContainerDeletionTime(long latestContainerDeletionTimestamp) throws CloudStorageException {
    try {
        ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
        buffer.putLong(latestContainerDeletionTimestamp);
        ByteArrayInputStream bais = new ByteArrayInputStream(buffer.array());
        requestAgent.doWithRetries(() -> {
            azureBlobDataAccessor.uploadFile(AzureCloudDestination.CHECKPOINT_CONTAINER, CONTAINER_DELETION_CHECKPOINT_FILE, bais);
            return null;
        }, "update-container-deletion-checkpoint", null);
    } catch (CloudStorageException e) {
        logger.error("Could not save update deprecated container progress", e);
        throw e;
    }
}

Example 7 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureCloudDestination method getBlobMetadata.

@Override
public Map<String, CloudBlobMetadata> getBlobMetadata(List<BlobId> blobIds) throws CloudStorageException {
    Objects.requireNonNull(blobIds, "blobIds cannot be null");
    if (blobIds.isEmpty()) {
        return Collections.emptyMap();
    }
    // needs to include that store key to replay the upload.
    if (!isVcr && blobIds.size() == 1) {
        CloudBlobMetadata metadata = azureBlobDataAccessor.getBlobMetadata(blobIds.get(0));
        return metadata == null ? Collections.emptyMap() : Collections.singletonMap(metadata.getId(), metadata);
    }
    // CosmosDB has query size limit of 256k chars.
    // Break list into chunks if necessary to avoid overflow.
    List<CloudBlobMetadata> metadataList = new ArrayList<>();
    List<List<BlobId>> chunkedBlobIdList = Utils.partitionList(blobIds, queryBatchSize);
    for (List<BlobId> batchOfBlobs : chunkedBlobIdList) {
        metadataList.addAll(getBlobMetadataChunked(batchOfBlobs));
    }
    return metadataList.stream().collect(Collectors.toMap(CloudBlobMetadata::getId, Function.identity(), (x, y) -> x));
}

Example 8 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureCloudDestinationTest method testFindEntriesSinceUsingChangeFeedWithInvalidToken.

/**
 * Query changefeed with invalid token and check for cache miss.
 * @param findToken {@link CosmosChangeFeedFindToken} to continue from.
 * @throws CloudStorageException
 */
private void testFindEntriesSinceUsingChangeFeedWithInvalidToken(CosmosChangeFeedFindToken findToken) throws CloudStorageException {
    // Invalid session id.
    CosmosChangeFeedFindToken invalidFindToken = new CosmosChangeFeedFindToken(findToken.getBytesRead(), findToken.getStartContinuationToken(), findToken.getEndContinuationToken(), findToken.getIndex(), findToken.getTotalItems(), UUID.randomUUID().toString(), findToken.getVersion());
    FindResult findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), invalidFindToken, 10);
    findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
    assertEquals("Unexpected change feed cache miss count", 2, azureMetrics.changeFeedCacheMissRate.getCount());
    assertEquals("Unexpected change feed cache refresh count", 1, azureMetrics.changeFeedCacheRefreshRate.getCount());
    assertEquals("Unexpected change feed cache hit count", 3, azureMetrics.changeFeedCacheHitRate.getCount());
    // invalid end token.
    invalidFindToken = new CosmosChangeFeedFindToken(findToken.getBytesRead(), findToken.getStartContinuationToken(), "5000", findToken.getIndex(), findToken.getTotalItems(), findToken.getCacheSessionId(), findToken.getVersion());
    findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), invalidFindToken, 10);
    findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
    assertEquals("Unexpected change feed cache miss count", 3, azureMetrics.changeFeedCacheMissRate.getCount());
    assertEquals("Unexpected change feed cache refresh count", 1, azureMetrics.changeFeedCacheRefreshRate.getCount());
    assertEquals("Unexpected change feed cache hit count", 3, azureMetrics.changeFeedCacheHitRate.getCount());
    // invalid start token.
    invalidFindToken = new CosmosChangeFeedFindToken(findToken.getBytesRead(), "5000", findToken.getEndContinuationToken(), findToken.getIndex(), findToken.getTotalItems(), findToken.getCacheSessionId(), findToken.getVersion());
    try {
        azureDest.findEntriesSince(blobId.getPartition().toPathString(), invalidFindToken, 10);
    } catch (Exception ex) {
    }
    assertEquals("Unexpected change feed cache miss count", 4, azureMetrics.changeFeedCacheMissRate.getCount());
    assertEquals("Unexpected change feed cache refresh count", 1, azureMetrics.changeFeedCacheRefreshRate.getCount());
    assertEquals("Unexpected change feed cache hit count", 3, azureMetrics.changeFeedCacheHitRate.getCount());
    // invalid total items.
    invalidFindToken = new CosmosChangeFeedFindToken(findToken.getBytesRead(), findToken.getStartContinuationToken(), findToken.getEndContinuationToken(), findToken.getIndex(), 9000, findToken.getCacheSessionId(), findToken.getVersion());
    azureDest.findEntriesSince(blobId.getPartition().toPathString(), invalidFindToken, 10);
    assertEquals("Unexpected change feed cache miss count", 5, azureMetrics.changeFeedCacheMissRate.getCount());
    assertEquals("Unexpected change feed cache refresh count", 1, azureMetrics.changeFeedCacheRefreshRate.getCount());
    assertEquals("Unexpected change feed cache hit count", 3, azureMetrics.changeFeedCacheHitRate.getCount());
}

Example 9 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureCloudDestinationTest method testDeleteAfterPartialCompaction.

/**
 * Test delete when record is in Cosmos but not ABS.
 */
@Test
public void testDeleteAfterPartialCompaction() throws Exception {
    BlobStorageException ex = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenThrow(ex);
    // Rig Cosmos to return us a deleted blob on read request
    long deletionTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10);
    CloudBlobMetadata deletedMetadata = new CloudBlobMetadata().setId(blobId.getID()).setDeletionTime(deletionTime);
    Observable<ResourceResponse<Document>> mockResponse = getMockedObservableForSingleResource(deletedMetadata);
    when(mockumentClient.readDocument(anyString(), any(RequestOptions.class))).thenReturn(mockResponse);
    // Now delete the puppy, Cosmos record should get purged.
    try {
        assertFalse("Expected update to recover and return false", azureDest.deleteBlob(blobId, deletionTime, (short) 0, dummyCloudUpdateValidator));
    } catch (CloudStorageException cex) {
        assertTrue(cex.getCause() instanceof BlobStorageException);
        assertEquals(((BlobStorageException) cex.getCause()).getErrorCode(), BlobErrorCode.BLOB_NOT_FOUND);
    }
    assertEquals("Expected recovery", 1, azureMetrics.blobUpdateRecoverCount.getCount());
    verify(mockumentClient).deleteDocument(anyString(), any());
}

Example 10 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureIntegrationTest method testNormalFlow.

/**
 * Test normal operations.
 * @throws Exception on error
 */
@Test
public void testNormalFlow() throws Exception {
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    BlobId blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
    byte[] uploadData = TestUtils.getRandomBytes(blobSize);
    InputStream inputStream = new ByteArrayInputStream(uploadData);
    long now = System.currentTimeMillis();
    CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, now, now + 60000, blobSize, CloudBlobMetadata.EncryptionOrigin.VCR, vcrKmsContext, cryptoAgentFactory, blobSize, (short) 0);
    // attempt undelete before uploading blob
    try {
        undeleteBlobWithRetry(blobId, (short) 1);
        fail("Undelete of a non existent blob should fail.");
    } catch (CloudStorageException cex) {
        assertEquals(cex.getStatusCode(), HttpConstants.StatusCodes.NOTFOUND);
    }
    assertTrue("Expected upload to return true", AzureTestUtils.uploadBlobWithRetry(blobId, blobSize, cloudBlobMetadata, inputStream, cloudRequestAgent, azureDest));
    // Get blob should return the same data
    verifyDownloadMatches(blobId, uploadData);
    // Try to upload same blob again
    assertFalse("Expected duplicate upload to return false", AzureTestUtils.uploadBlobWithRetry(blobId, blobSize, cloudBlobMetadata, new ByteArrayInputStream(uploadData), cloudRequestAgent, azureDest));
    // ttl update
    long expirationTime = Utils.Infinite_Time;
    try {
        updateBlobExpirationWithRetry(blobId, expirationTime);
    } catch (Exception ex) {
        fail("Expected update to be successful");
    }
    CloudBlobMetadata metadata = getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).get(blobId.getID());
    assertEquals(expirationTime, metadata.getExpirationTime());
    // delete blob
    long deletionTime = now + 10000;
    // TODO add a test case here to verify life version after delete.
    assertTrue("Expected deletion to return true", cloudRequestAgent.doWithRetries(() -> azureDest.deleteBlob(blobId, deletionTime, (short) 0, dummyCloudUpdateValidator), "DeleteBlob", partitionId.toPathString()));
    metadata = getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).get(blobId.getID());
    assertEquals(deletionTime, metadata.getDeletionTime());
    // undelete blob
    assertEquals(undeleteBlobWithRetry(blobId, (short) 1), 1);
    metadata = getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).get(blobId.getID());
    assertEquals(metadata.getDeletionTime(), Utils.Infinite_Time);
    assertEquals(metadata.getLifeVersion(), 1);
    // undelete with a higher life version updates life version.
    assertEquals(undeleteBlobWithRetry(blobId, (short) 2), 2);
    metadata = getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).get(blobId.getID());
    assertEquals(metadata.getDeletionTime(), Utils.Infinite_Time);
    assertEquals(metadata.getLifeVersion(), 2);
    // delete after undelete.
    long newDeletionTime = now + 20000;
    // TODO add a test case here to verify life version after delete.
    assertTrue("Expected deletion to return true", cloudRequestAgent.doWithRetries(() -> azureDest.deleteBlob(blobId, newDeletionTime, (short) 3, dummyCloudUpdateValidator), "DeleteBlob", partitionId.toPathString()));
    metadata = getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).get(blobId.getID());
    assertEquals(newDeletionTime, metadata.getDeletionTime());
    // delete changes life version.
    assertEquals(metadata.getLifeVersion(), 3);
    // compact partition
    azureDest.compactPartition(partitionId.toPathString());
    assertTrue("Expected empty set after purge", getBlobMetadataWithRetry(Collections.singletonList(blobId), partitionId.toPathString(), cloudRequestAgent, azureDest).isEmpty());
    // Get blob should fail after purge
    try {
        verifyDownloadMatches(blobId, uploadData);
        fail("download blob should fail after data is purged");
    } catch (CloudStorageException csex) {
    }
}