Examples with CloudStorageException com.github.ambry.cloud.CloudStorageException used on opensource projects

Example 1 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureContainerCompactorIntegrationTest method cleanup.

/**
 * Cleanup entries in the cosmos db container deletion table.
 * @throws DocumentClientException in case of any exception.
 */
private void cleanup() throws DocumentClientException {
    ConnectionPolicy connectionPolicy = new ConnectionPolicy();
    connectionPolicy.setRequestTimeoutInMillis(cloudConfig.cloudQueryRequestTimeout);
    // Note: retry decisions are made at CloudBlobStore level.  Configure Cosmos with no retries.
    RetryOptions noRetries = new RetryOptions();
    noRetries.setMaxRetryAttemptsOnThrottledRequests(0);
    connectionPolicy.setRetryOptions(noRetries);
    if (azureCloudConfig.cosmosDirectHttps) {
        logger.info("Using CosmosDB DirectHttps connection mode");
        connectionPolicy.setConnectionMode(ConnectionMode.Direct);
    }
    AsyncDocumentClient asyncDocumentClient = new AsyncDocumentClient.Builder().withServiceEndpoint(azureCloudConfig.cosmosEndpoint).withMasterKeyOrResourceToken(azureCloudConfig.cosmosKey).withConnectionPolicy(connectionPolicy).withConsistencyLevel(ConsistencyLevel.Session).build();
    Set<CosmosContainerDeletionEntry> entries = cloudDestination.getCosmosDataAccessor().getDeprecatedContainers(100);
    AtomicBoolean error = new AtomicBoolean(false);
    while (!entries.isEmpty() && !error.get()) {
        entries.stream().forEach(entry -> {
            try {
                RequestOptions requestOptions = new RequestOptions();
                requestOptions.setPartitionKey(new PartitionKey(entry.getId()));
                cloudRequestAgent.doWithRetries(() -> CosmosDataAccessor.executeCosmosAction(() -> asyncDocumentClient.deleteDocument(azureCloudConfig.cosmosDeletedContainerCollectionLink + "/docs/" + entry.getId(), requestOptions).toBlocking().single(), null), "Test Cleanup", entry.getId());
            } catch (CloudStorageException ex) {
                logger.warn("Failed to delete container deprecation entry {}. Unable to cleanup", entry);
                error.set(true);
            }
        });
        entries = cloudDestination.getCosmosDataAccessor().getDeprecatedContainers(100);
    }
}

Example 2 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureIntegrationTest method testConcurrentUpdates.

/**
 * Test that concurrent updates fail when the precondition does not match.
 * We don't test retries here since CloudBlobStoreTest covers that.
 */
@Test
public void testConcurrentUpdates() throws Exception {
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    BlobId blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
    InputStream inputStream = getBlobInputStream(blobSize);
    long now = System.currentTimeMillis();
    CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, now, now + 60000, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
    uploadBlobWithRetry(blobId, blobSize, cloudBlobMetadata, inputStream, cloudRequestAgent, azureDest);
    // Different instance to simulate concurrent update in separate session.
    AzureCloudDestination concurrentUpdater = getAzureDestination(verifiableProperties);
    String fieldName = CloudBlobMetadata.FIELD_UPLOAD_TIME;
    long newUploadTime = now++;
    // Case 1: concurrent modification to blob metadata.
    azureDest.getAzureBlobDataAccessor().setUpdateCallback(() -> concurrentUpdater.getAzureBlobDataAccessor().updateBlobMetadata(blobId, Collections.singletonMap(fieldName, newUploadTime), dummyCloudUpdateValidator));
    try {
        azureDest.updateBlobExpiration(blobId, ++now, dummyCloudUpdateValidator);
        fail("Expected 412 error");
    } catch (CloudStorageException csex) {
        // TODO: check nested exception is BlobStorageException with status code 412
        assertEquals("Expected update conflict", 1, azureDest.getAzureMetrics().blobUpdateConflictCount.getCount());
    }
    // Case 2: concurrent modification to Cosmos record.
    azureDest.getCosmosDataAccessor().setUpdateCallback(() -> concurrentUpdater.getCosmosDataAccessor().updateMetadata(blobId, Collections.singletonMap(fieldName, Long.toString(newUploadTime))));
    try {
        azureDest.updateBlobExpiration(blobId, ++now, dummyCloudUpdateValidator);
        fail("Expected 412 error");
    } catch (CloudStorageException csex) {
        assertEquals("Expected update conflict", 2, azureDest.getAzureMetrics().blobUpdateConflictCount.getCount());
    }
    azureDest.getCosmosDataAccessor().setUpdateCallback(null);
    try {
        azureDest.updateBlobExpiration(blobId, ++now, dummyCloudUpdateValidator);
    } catch (Exception ex) {
        fail("Expected update to succeed.");
    }
    assertEquals("Expected no new update conflict", 2, azureDest.getAzureMetrics().blobUpdateConflictCount.getCount());
}

Example 3 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureIntegrationTest method testRepairAfterIncompleteCompaction.

/**
 * Test that incomplete compaction get fixed on update.
 */
@Test
public void testRepairAfterIncompleteCompaction() throws Exception {
    // Upload a blob
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    BlobId blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
    InputStream inputStream = getBlobInputStream(blobSize);
    long now = System.currentTimeMillis();
    CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, now, -1, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
    uploadBlobWithRetry(blobId, blobSize, cloudBlobMetadata, inputStream, cloudRequestAgent, azureDest);
    // Mark it deleted in the past
    long deletionTime = now - TimeUnit.DAYS.toMillis(7);
    assertTrue("Expected delete to return true", azureDest.deleteBlob(blobId, deletionTime, (short) 0, dummyCloudUpdateValidator));
    // Simulate incomplete compaction by purging it from ABS only
    azureDest.getAzureBlobDataAccessor().purgeBlobs(Collections.singletonList(cloudBlobMetadata));
    // Try to delete again (to trigger recovery), verify removed from Cosmos
    try {
        azureDest.deleteBlob(blobId, deletionTime, (short) 0, dummyCloudUpdateValidator);
    } catch (CloudStorageException cex) {
        assertEquals("Unexpected error code", HttpConstants.StatusCodes.NOTFOUND, cex.getStatusCode());
    }
    assertNull("Expected record to be purged from Cosmos", azureDest.getCosmosDataAccessor().getMetadataOrNull(blobId));
}

Example 4 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureStorageCompactor method updateCompactionProgress.

/**
 * Update the compaction progress for a partition.
 * @param partitionPath the partition to update.
 * @param fieldName the compaction field (deletion or expiration time).
 * @param progressTime the updated progress time.
 * @return true if the checkpoint file was updated, otherwise false.
 */
boolean updateCompactionProgress(String partitionPath, String fieldName, long progressTime) {
    try {
        // load existing progress checkpoint.
        Map<String, Long> checkpoints = getCompactionProgress(partitionPath);
        // Ensure we don't downgrade progress already recorded.
        if (progressTime <= checkpoints.getOrDefault(fieldName, DEFAULT_TIME)) {
            logger.info("Skipping update of compaction progress for {} because saved {} is more recent.", partitionPath, fieldName);
            return false;
        }
        checkpoints.put(fieldName, Math.max(progressTime, checkpoints.get(fieldName)));
        String json = objectMapper.writeValueAsString(checkpoints);
        ByteArrayInputStream bais = new ByteArrayInputStream(json.getBytes());
        requestAgent.doWithRetries(() -> {
            azureBlobDataAccessor.uploadFile(AzureCloudDestination.CHECKPOINT_CONTAINER, partitionPath, bais);
            return null;
        }, "Update compaction progress", partitionPath);
        logger.info("Marked compaction of partition {} complete up to {} {}", partitionPath, fieldName, new Date(progressTime));
        return true;
    } catch (CloudStorageException | IOException e) {
        logger.error("Could not save compaction progress for {}", partitionPath, e);
        azureMetrics.compactionProgressWriteErrorCount.inc();
        return false;
    }
}

Example 5 with CloudStorageException

use of com.github.ambry.cloud.CloudStorageException in project ambry by linkedin.

the class AzureStorageCompactor method getAllCompactionProgress.

/**
 * Retrieve the compaction progress for all partitions, sorted by furthest behind.
 * @return a list of pairs each containing a partition and its latest progress time.
 * @throws Exception
 */
List<Pair<String, Long>> getAllCompactionProgress() throws Exception {
    // Read all checkpoint files and dump results into sortable table.
    BlobContainerClient containerClient = azureBlobDataAccessor.getStorageClient().getBlobContainerClient(AzureCloudDestination.CHECKPOINT_CONTAINER);
    List<String> checkpoints = new ArrayList<>();
    containerClient.listBlobs().forEach(item -> {
        checkpoints.add(item.getName());
    });
    logger.info("Retrieving checkpoints for {} partitions", checkpoints.size());
    List<Pair<String, Long>> partitionProgressList = Collections.synchronizedList(new ArrayList<>());
    ForkJoinPool forkJoinPool = new ForkJoinPool(numThreads);
    forkJoinPool.submit(() -> {
        checkpoints.parallelStream().forEach(partition -> {
            try {
                Map<String, Long> map = getCompactionProgress(partition);
                long progressTime = Collections.min(map.values());
                partitionProgressList.add(new Pair(partition, progressTime));
            } catch (CloudStorageException cse) {
                logger.error("Failed for partition {}", partition, cse);
            }
        });
    }).get();
    Collections.sort(partitionProgressList, Comparator.comparingLong(Pair::getSecond));
    return partitionProgressList;
}