Examples with CloudBlobMetadata com.github.ambry.cloud.CloudBlobMetadata used on opensource projects

Example 21 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CosmosDataAccessor method getDeadBlobs.

/**
 * Get the list of blobs in the specified partition that have been deleted or expired for at least the
 * configured retention period.
 * @param partitionPath the partition to query.
 * @param fieldName the field name to query on. Allowed values are {@link CloudBlobMetadata#FIELD_DELETION_TIME} and
 *                  {@link CloudBlobMetadata#FIELD_EXPIRATION_TIME}.
 * @param startTime the start of the query time range.
 * @param endTime the end of the query time range.
 * @param maxEntries the max number of metadata records to return.
 * @return a List of {@link CloudBlobMetadata} referencing the dead blobs found.
 * @throws DocumentClientException
 */
List<CloudBlobMetadata> getDeadBlobs(String partitionPath, String fieldName, long startTime, long endTime, int maxEntries) throws DocumentClientException {
    String deadBlobsQuery;
    if (fieldName.equals(CloudBlobMetadata.FIELD_DELETION_TIME)) {
        deadBlobsQuery = DELETED_BLOBS_QUERY;
    } else if (fieldName.equals(CloudBlobMetadata.FIELD_EXPIRATION_TIME)) {
        deadBlobsQuery = EXPIRED_BLOBS_QUERY;
    } else {
        throw new IllegalArgumentException("Invalid field: " + fieldName);
    }
    SqlQuerySpec querySpec = new SqlQuerySpec(deadBlobsQuery, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries), new SqlParameter(START_TIME_PARAM, startTime), new SqlParameter(END_TIME_PARAM, endTime)));
    FeedOptions feedOptions = new FeedOptions();
    feedOptions.setMaxItemCount(maxEntries);
    feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
    feedOptions.setPartitionKey(new PartitionKey(partitionPath));
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, azureMetrics.deadBlobsQueryTime).getIterator();
        List<CloudBlobMetadata> deadBlobsList = new ArrayList<>();
        double requestCharge = 0.0;
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            requestCharge += response.getRequestCharge();
            response.getResults().iterator().forEachRemaining(doc -> deadBlobsList.add(createMetadataFromDocument(doc)));
        }
        if (requestCharge >= requestChargeThreshold) {
            logger.info("Dead blobs query partition {} endTime {} request charge {} for {} records", partitionPath, new Date(endTime), requestCharge, deadBlobsList.size());
        }
        return deadBlobsList;
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Dead blobs query {} partition {} got {}", deadBlobsQuery, partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
}

Example 22 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class StorageClient method deleteBatchAsync.

/**
 * Deletes a list of blobs asynchronously. This method will block for now until classes like
 * {@link com.github.ambry.cloud.CloudDestination} are refactored to handle responses asynchronously.
 * @param batchOfBlobs {@link List} of {@link CloudBlobMetadata} objects.
 * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised.
 * @return {@link List} of {@link Response}s for the blobs in the batch.
 * @throws RuntimeException If the {@code timeout} duration completes before a response is returned.
 * @throws BlobStorageException If the batch request is malformed.
 * @throws BlobBatchStorageException If {@code throwOnAnyFailure} is {@code true} and any request in the
 * {@link BlobBatch} failed.
 */
public List<Response<Void>> deleteBatchAsync(List<CloudBlobMetadata> batchOfBlobs, Duration timeout) {
    List<Response<Void>> responseList = new ArrayList<>();
    doStorageClientOperation(() -> {
        BlobBatch blobBatch = blobBatchAsyncClientRef.get().getBlobBatch();
        for (CloudBlobMetadata blobMetadata : batchOfBlobs) {
            AzureBlobLayoutStrategy.BlobLayout blobLayout = blobLayoutStrategy.getDataBlobLayout(blobMetadata);
            responseList.add(blobBatch.deleteBlob(blobLayout.containerName, blobLayout.blobFilePath));
        }
        blobBatchAsyncClientRef.get().submitBatchWithResponse(blobBatch, false).toFuture().get(timeout.toMillis(), TimeUnit.MILLISECONDS);
        return null;
    });
    return responseList;
}

Example 23 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class StorageClient method deleteBatch.

/**
 * Deletes a list of blobs.
 * @param batchOfBlobs {@link List} of {@link CloudBlobMetadata} objects.
 * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised.
 * @return {@link List} of {@link Response}s for the blobs in the batch.
 * @throws RuntimeException If the {@code timeout} duration completes before a response is returned.
 * @throws BlobStorageException If the batch request is malformed.
 * @throws BlobBatchStorageException If {@code throwOnAnyFailure} is {@code true} and any request in the
 * {@link BlobBatch} failed.
 */
public List<Response<Void>> deleteBatch(List<CloudBlobMetadata> batchOfBlobs, Duration timeout) {
    if (azureCloudConfig.useAsyncAzureAPIs) {
        return deleteBatchAsync(batchOfBlobs, timeout);
    }
    List<Response<Void>> responseList = new ArrayList<>();
    doStorageClientOperation(() -> {
        BlobBatch blobBatch = blobBatchClientRef.get().getBlobBatch();
        for (CloudBlobMetadata blobMetadata : batchOfBlobs) {
            AzureBlobLayoutStrategy.BlobLayout blobLayout = blobLayoutStrategy.getDataBlobLayout(blobMetadata);
            responseList.add(blobBatch.deleteBlob(blobLayout.containerName, blobLayout.blobFilePath));
        }
        blobBatchClientRef.get().submitBatchWithResponse(blobBatch, false, timeout, Context.NONE);
        return null;
    });
    return responseList;
}

Example 24 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobMetadataTest method testEncrypted.

/**
 * Encrypted blob
 */
@Test
public void testEncrypted() throws Exception {
    // Router encrypted
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, now, -1, 1024, EncryptionOrigin.ROUTER);
    verifySerde(blobMetadata, ArrayUtils.addAll(FIELDS_ALWAYS_SET, FIELD_ENCRYPTION_ORIGIN), ArrayUtils.removeElement(FIELDS_RARELY_SET, FIELD_ENCRYPTION_ORIGIN));
    blobMetadata = new CloudBlobMetadata(blobId, now, -1, 1024, EncryptionOrigin.VCR, "context", "factory", 1056, (short) 0);
    verifySerde(blobMetadata, ArrayUtils.addAll(FIELDS_ALWAYS_SET, ENCRYPTION_FIELDS), new String[] { FIELD_DELETION_TIME, FIELD_EXPIRATION_TIME });
}

Example 25 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobMetadataTest method testDeleted.

/**
 * Deleted blob
 */
@Test
public void testDeleted() throws Exception {
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, now, -1, 1024, EncryptionOrigin.NONE);
    blobMetadata.setDeletionTime(futureTime);
    verifySerde(blobMetadata, ArrayUtils.addAll(FIELDS_ALWAYS_SET, FIELD_DELETION_TIME), ArrayUtils.removeElement(FIELDS_RARELY_SET, FIELD_DELETION_TIME));
}