Examples with BlobBatch com.azure.storage.blob.batch.BlobBatch used on opensource projects

Example 1 with BlobBatch

use of com.azure.storage.blob.batch.BlobBatch in project ambry by linkedin.

the class AzureStorageCompactorTest method buildCompactor.

private void buildCompactor(Properties configProps) throws Exception {
    CloudConfig cloudConfig = new CloudConfig(new VerifiableProperties(configProps));
    VcrMetrics vcrMetrics = new VcrMetrics(new MetricRegistry());
    azureBlobDataAccessor = new AzureBlobDataAccessor(mockServiceClient, mockBlobBatchClient, clusterName, azureMetrics, new AzureCloudConfig(new VerifiableProperties(configProps)));
    cosmosDataAccessor = new CosmosDataAccessor(mockumentClient, collectionLink, cosmosDeletedContainerCollectionLink, vcrMetrics, azureMetrics);
    azureStorageCompactor = new AzureStorageCompactor(azureBlobDataAccessor, cosmosDataAccessor, cloudConfig, vcrMetrics, azureMetrics);
    // Mocks for getDeadBlobs query
    List<Document> docList = new ArrayList<>();
    for (int j = 0; j < numBlobsPerQuery; j++) {
        BlobId blobId = generateBlobId();
        CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, testTime, Utils.Infinite_Time, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
        blobMetadataList.add(inputMetadata);
        docList.add(AzureTestUtils.createDocumentFromCloudBlobMetadata(inputMetadata));
    }
    Observable<FeedResponse<Document>> mockResponse = mock(Observable.class);
    mockObservableForQuery(docList, mockResponse);
    when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(mockResponse);
    // Mocks for purge
    BlobBatch mockBatch = mock(BlobBatch.class);
    when(mockBlobBatchClient.getBlobBatch()).thenReturn(mockBatch);
    Response<Void> okResponse = mock(Response.class);
    when(okResponse.getStatusCode()).thenReturn(202);
    when(mockBatch.deleteBlob(anyString(), anyString())).thenReturn(okResponse);
    Observable<StoredProcedureResponse> mockBulkDeleteResponse = getMockBulkDeleteResponse(1);
    when(mockumentClient.executeStoredProcedure(anyString(), any(RequestOptions.class), any())).thenReturn(mockBulkDeleteResponse);
    String checkpointJson = objectMapper.writeValueAsString(AzureStorageCompactor.emptyCheckpoints);
    mockCheckpointDownload(true, checkpointJson);
}

Example 2 with BlobBatch

use of com.azure.storage.blob.batch.BlobBatch in project ambry by linkedin.

the class StorageClient method deleteBatchAsync.

/**
 * Deletes a list of blobs asynchronously. This method will block for now until classes like
 * {@link com.github.ambry.cloud.CloudDestination} are refactored to handle responses asynchronously.
 * @param batchOfBlobs {@link List} of {@link CloudBlobMetadata} objects.
 * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised.
 * @return {@link List} of {@link Response}s for the blobs in the batch.
 * @throws RuntimeException If the {@code timeout} duration completes before a response is returned.
 * @throws BlobStorageException If the batch request is malformed.
 * @throws BlobBatchStorageException If {@code throwOnAnyFailure} is {@code true} and any request in the
 * {@link BlobBatch} failed.
 */
public List<Response<Void>> deleteBatchAsync(List<CloudBlobMetadata> batchOfBlobs, Duration timeout) {
    List<Response<Void>> responseList = new ArrayList<>();
    doStorageClientOperation(() -> {
        BlobBatch blobBatch = blobBatchAsyncClientRef.get().getBlobBatch();
        for (CloudBlobMetadata blobMetadata : batchOfBlobs) {
            AzureBlobLayoutStrategy.BlobLayout blobLayout = blobLayoutStrategy.getDataBlobLayout(blobMetadata);
            responseList.add(blobBatch.deleteBlob(blobLayout.containerName, blobLayout.blobFilePath));
        }
        blobBatchAsyncClientRef.get().submitBatchWithResponse(blobBatch, false).toFuture().get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        return null;
    });
    return responseList;
}

Example 3 with BlobBatch

use of com.azure.storage.blob.batch.BlobBatch in project ambry by linkedin.

the class StorageClient method deleteBatch.

/**
 * Deletes a list of blobs.
 * @param batchOfBlobs {@link List} of {@link CloudBlobMetadata} objects.
 * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised.
 * @return {@link List} of {@link Response}s for the blobs in the batch.
 * @throws RuntimeException If the {@code timeout} duration completes before a response is returned.
 * @throws BlobStorageException If the batch request is malformed.
 * @throws BlobBatchStorageException If {@code throwOnAnyFailure} is {@code true} and any request in the
 * {@link BlobBatch} failed.
 */
public List<Response<Void>> deleteBatch(List<CloudBlobMetadata> batchOfBlobs, Duration timeout) {
    if (azureCloudConfig.useAsyncAzureAPIs) {
        return deleteBatchAsync(batchOfBlobs, timeout);
    }
    List<Response<Void>> responseList = new ArrayList<>();
    doStorageClientOperation(() -> {
        BlobBatch blobBatch = blobBatchClientRef.get().getBlobBatch();
        for (CloudBlobMetadata blobMetadata : batchOfBlobs) {
            AzureBlobLayoutStrategy.BlobLayout blobLayout = blobLayoutStrategy.getDataBlobLayout(blobMetadata);
            responseList.add(blobBatch.deleteBlob(blobLayout.containerName, blobLayout.blobFilePath));
        }
        blobBatchClientRef.get().submitBatchWithResponse(blobBatch, false, timeout, Context.NONE);
        return null;
    });
    return responseList;
}

Example 4 with BlobBatch

use of com.azure.storage.blob.batch.BlobBatch in project ambry by linkedin.

the class AzureBlobDataAccessorTest method testPurge.

/**
 * Test purge
 */
@Test
public void testPurge() throws Exception {
    // purge 3 blobs, response status (202, 404, 503)
    String blobNameOkStatus = "andromeda";
    String blobNameNotFoundStatus = "sirius";
    String blobNameErrorStatus = "mutant";
    BlobBatch mockBatch = mock(BlobBatch.class);
    when(mockBatchClient.getBlobBatch()).thenReturn(mockBatch);
    Response<Void> okResponse = mock(Response.class);
    when(okResponse.getStatusCode()).thenReturn(202);
    when(mockBatch.deleteBlob(anyString(), endsWith(blobNameOkStatus))).thenReturn(okResponse);
    BlobStorageException notFoundException = mock(BlobStorageException.class);
    when(notFoundException.getStatusCode()).thenReturn(404);
    Response<Void> notFoundResponse = mock(Response.class);
    when(notFoundResponse.getStatusCode()).thenThrow(notFoundException);
    when(mockBatch.deleteBlob(anyString(), endsWith(blobNameNotFoundStatus))).thenReturn(notFoundResponse);
    BlobStorageException badException = mock(BlobStorageException.class);
    when(badException.getStatusCode()).thenReturn(503);
    Response<Void> badResponse = mock(Response.class);
    when(badResponse.getStatusCode()).thenThrow(badException);
    when(mockBatch.deleteBlob(anyString(), endsWith(blobNameErrorStatus))).thenReturn(badResponse);
    List<CloudBlobMetadata> purgeList = new ArrayList<>();
    purgeList.add(new CloudBlobMetadata().setId(blobNameOkStatus));
    purgeList.add(new CloudBlobMetadata().setId(blobNameNotFoundStatus));
    // Purge first 2 and expect success
    List<CloudBlobMetadata> purgeResponseList = dataAccessor.purgeBlobs(purgeList);
    assertEquals("Wrong response size", 2, purgeResponseList.size());
    assertEquals("Wrong blob name", blobNameOkStatus, purgeResponseList.get(0).getId());
    assertEquals("Wrong blob name", blobNameNotFoundStatus, purgeResponseList.get(1).getId());
    // Including last one should fail
    purgeList.add(new CloudBlobMetadata().setId(blobNameErrorStatus));
    try {
        dataAccessor.purgeBlobs(purgeList);
        fail("Expected purge to fail");
    } catch (BlobStorageException bex) {
        assertEquals("Unexpected status code", 503, bex.getStatusCode());
    }
}

Example 5 with BlobBatch

use of com.azure.storage.blob.batch.BlobBatch in project ambry by linkedin.

the class AzureStorageCompactorTest method testPurgeWithStorageError.

/**
 * Test purgeBlobs with ABS error
 */
@Test
public void testPurgeWithStorageError() throws Exception {
    // Unsuccessful case
    BlobStorageException ex = mockStorageException(BlobErrorCode.BLOB_ARCHIVED);
    BlobBatch mockBatch = mock(BlobBatch.class);
    Response<Void> mockResponse = mock(Response.class);
    when(mockResponse.getStatusCode()).thenThrow(ex);
    when(mockBlobBatchClient.getBlobBatch()).thenReturn(mockBatch);
    when(mockBatch.deleteBlob(anyString(), anyString())).thenReturn(mockResponse);
    try {
        AzureCompactionUtil.purgeBlobs(blobMetadataList, azureBlobDataAccessor, azureMetrics, cosmosDataAccessor);
        fail("Expected CloudStorageException");
    } catch (CloudStorageException bex) {
    }
    assertEquals(0, azureMetrics.blobDeletedCount.getCount());
    assertEquals(numBlobsPerQuery, azureMetrics.blobDeleteErrorCount.getCount());
}