Examples with FeedResponse com.microsoft.azure.cosmosdb.FeedResponse used on opensource projects

Example 6 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessor method getDeprecatedContainers.

/**
 * Fetch a {@link Set} of {@link CosmosContainerDeletionEntry} objects from cosmos db that are not marked as deleted.
 * @param maxEntries Max number of entries to fetch on one query.
 * @return {@link Set} of {@link CosmosContainerDeletionEntry} objects.
 * @throws DocumentClientException in case of any error.
 */
public Set<CosmosContainerDeletionEntry> getDeprecatedContainers(int maxEntries) throws DocumentClientException {
    SqlQuerySpec querySpec = new SqlQuerySpec(DEPRECATED_CONTAINERS_QUERY, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries)));
    Timer timer = new Timer();
    Set<CosmosContainerDeletionEntry> containerDeletionEntries = new HashSet<>();
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(cosmosDeletedContainerCollectionLink, null, querySpec, new FeedOptions(), timer).getIterator();
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            response.getResults().iterator().forEachRemaining(doc -> containerDeletionEntries.add(CosmosContainerDeletionEntry.fromJson(new JSONObject(doc.toJson()))));
        }
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Get deprecated containers query {} got {}", querySpec.getQueryText(), ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
    return containerDeletionEntries;
}

Example 7 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessor method getContainerBlobs.

/**
 * Get the list of blobs in the specified partition that belong to the specified container.
 * @param partitionPath the partition to query.
 * @param accountId account id of the container.
 * @param containerId container id of the container.
 * @param queryLimit max number of blobs to return.
 * @return a List of {@link CloudBlobMetadata} referencing the blobs belonging to the deprecated containers.
 * @throws DocumentClientException in case of any error.
 */
List<CloudBlobMetadata> getContainerBlobs(String partitionPath, short accountId, short containerId, int queryLimit) throws DocumentClientException {
    SqlQuerySpec querySpec = new SqlQuerySpec(CONTAINER_BLOBS_QUERY, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, queryLimit), new SqlParameter(CONTAINER_ID_PARAM, containerId), new SqlParameter(ACCOUNT_ID_PARAM, accountId)));
    FeedOptions feedOptions = new FeedOptions();
    feedOptions.setMaxItemCount(queryLimit);
    feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
    feedOptions.setPartitionKey(new PartitionKey(partitionPath));
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, azureMetrics.deletedContainerBlobsQueryTime).getIterator();
        List<CloudBlobMetadata> containerBlobsList = new ArrayList<>();
        double requestCharge = 0.0;
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            requestCharge += response.getRequestCharge();
            response.getResults().iterator().forEachRemaining(doc -> containerBlobsList.add(createMetadataFromDocument(doc)));
        }
        if (requestCharge >= requestChargeThreshold) {
            logger.info("Deleted container blobs query partition {} containerId {} accountId {} request charge {} for {} records", partitionPath, containerId, accountId, requestCharge, containerBlobsList.size());
        }
        return containerBlobsList;
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Dead blobs query {} partition {} got {}", querySpec.getQueryText(), partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
}

Example 8 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessor method queryMetadata.

/**
 * Get the list of blobs in the specified partition matching the specified DocumentDB query spec.
 * @param partitionPath the partition to query.
 * @param querySpec the DocumentDB {@link SqlQuerySpec} to execute.
 * @param timer the {@link Timer} to use to record query time (excluding waiting).
 * @return a List of {@link CloudBlobMetadata} referencing the matching blobs.
 */
List<CloudBlobMetadata> queryMetadata(String partitionPath, SqlQuerySpec querySpec, Timer timer) throws DocumentClientException {
    FeedOptions feedOptions = new FeedOptions();
    // TODO: set maxItemCount
    feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
    feedOptions.setPartitionKey(new PartitionKey(partitionPath));
    // TODO: consolidate error count here
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, timer).getIterator();
        List<CloudBlobMetadata> metadataList = new ArrayList<>();
        double requestCharge = 0.0;
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            requestCharge += response.getRequestCharge();
            response.getResults().iterator().forEachRemaining(doc -> metadataList.add(createMetadataFromDocument(doc)));
        }
        if (requestCharge >= requestChargeThreshold) {
            logger.info("Query partition {} request charge {} for {} records", partitionPath, requestCharge, metadataList.size());
        }
        return metadataList;
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Query {} on partition {} got {}", querySpec.getQueryText(), partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
}

Example 9 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CloudTestUtil method cleanupPartition.

/**
 * Cleanup the specified partition in azure by deleting all the blobs of the partition.
 * @param azureCloudConfig Properties containing the credentials needed for connection to azure.
 * @param partitionId partition to be deleted.
 */
static void cleanupPartition(AzureCloudConfig azureCloudConfig, PartitionId partitionId) {
    ConnectionPolicy connectionPolicy = new ConnectionPolicy();
    AsyncDocumentClient asyncDocumentClient = new AsyncDocumentClient.Builder().withServiceEndpoint(azureCloudConfig.cosmosEndpoint).withMasterKeyOrResourceToken(azureCloudConfig.cosmosKey).withConnectionPolicy(connectionPolicy).withConsistencyLevel(ConsistencyLevel.Session).build();
    SqlQuerySpec sqlQuerySpec = new SqlQuerySpec("select * from c where c.partitionId=\"" + partitionId.toPathString() + "\"");
    FeedOptions feedOptions = new FeedOptions();
    feedOptions.setPartitionKey(new PartitionKey(partitionId.toPathString()));
    Iterator<FeedResponse<Document>> iterator = asyncDocumentClient.queryDocuments(azureCloudConfig.cosmosCollectionLink, sqlQuerySpec, feedOptions).toBlocking().getIterator();
    RequestOptions requestOptions = new RequestOptions();
    requestOptions.setPartitionKey(new PartitionKey(partitionId.toPathString()));
    while (iterator.hasNext()) {
        FeedResponse<Document> response = iterator.next();
        response.getResults().forEach(document -> asyncDocumentClient.deleteDocument(azureCloudConfig.cosmosCollectionLink + "/docs/" + document.getId(), requestOptions).toBlocking().single());
    }
}

Example 10 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class AzureCloudDestinationTest method testGetOneMetadata.

/**
 * Test to make sure that getting metadata for single blob calls ABS when not vcr and Cosmos when vcr.
 */
@Test
public void testGetOneMetadata() throws Exception {
    // 
    // Test 1: isVcr = false (already setup)
    // 
    // Get for existing blob
    Response<BlobProperties> mockResponse = mock(Response.class);
    BlobProperties mockProperties = mock(BlobProperties.class);
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, 0, -1, 0, null);
    Map<String, String> propertyMap = blobMetadata.toMap();
    when(mockProperties.getMetadata()).thenReturn(propertyMap);
    when(mockResponse.getValue()).thenReturn(mockProperties);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenReturn(mockResponse);
    List<BlobId> singleBlobList = Collections.singletonList(blobId);
    Map<String, CloudBlobMetadata> metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertEquals("Expected map of one", 1, metadataMap.size());
    verify(mockBlockBlobClient).getPropertiesWithResponse(any(), any(), any());
    verifyZeroInteractions(mockumentClient);
    // Get for nonexistent blob
    BlobStorageException ex = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenThrow(ex);
    metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertTrue("Expected empty map", metadataMap.isEmpty());
    verify(mockBlockBlobClient, times(2)).getPropertiesWithResponse(any(), any(), any());
    verifyZeroInteractions(mockumentClient);
    // 
    // Test 2: isVcr = true
    // 
    azureDest.close();
    azureDest = new AzureCloudDestination(mockServiceClient, mockBlobBatchClient, mockumentClient, "foo", "bar", clusterName, azureMetrics, defaultAzureReplicationFeedType, clusterMap, true, configProps);
    // Existing blob
    List<Document> docList = Collections.singletonList(createDocumentFromCloudBlobMetadata(blobMetadata));
    Observable<FeedResponse<Document>> feedResponse = mock(Observable.class);
    mockObservableForQuery(docList, feedResponse);
    when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(feedResponse);
    metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertEquals("Expected map of one", 1, metadataMap.size());
    verify(mockumentClient).queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class));
    verify(mockBlockBlobClient, times(2)).getPropertiesWithResponse(any(), any(), any());
}