Examples with FeedResponse com.microsoft.azure.cosmosdb.FeedResponse used on opensource projects

Example 1 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class AzureStorageCompactorTest method buildCompactor.

private void buildCompactor(Properties configProps) throws Exception {
    CloudConfig cloudConfig = new CloudConfig(new VerifiableProperties(configProps));
    VcrMetrics vcrMetrics = new VcrMetrics(new MetricRegistry());
    azureBlobDataAccessor = new AzureBlobDataAccessor(mockServiceClient, mockBlobBatchClient, clusterName, azureMetrics, new AzureCloudConfig(new VerifiableProperties(configProps)));
    cosmosDataAccessor = new CosmosDataAccessor(mockumentClient, collectionLink, cosmosDeletedContainerCollectionLink, vcrMetrics, azureMetrics);
    azureStorageCompactor = new AzureStorageCompactor(azureBlobDataAccessor, cosmosDataAccessor, cloudConfig, vcrMetrics, azureMetrics);
    // Mocks for getDeadBlobs query
    List<Document> docList = new ArrayList<>();
    for (int j = 0; j < numBlobsPerQuery; j++) {
        BlobId blobId = generateBlobId();
        CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, testTime, Utils.Infinite_Time, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
        blobMetadataList.add(inputMetadata);
        docList.add(AzureTestUtils.createDocumentFromCloudBlobMetadata(inputMetadata));
    }
    Observable<FeedResponse<Document>> mockResponse = mock(Observable.class);
    mockObservableForQuery(docList, mockResponse);
    when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(mockResponse);
    // Mocks for purge
    BlobBatch mockBatch = mock(BlobBatch.class);
    when(mockBlobBatchClient.getBlobBatch()).thenReturn(mockBatch);
    Response<Void> okResponse = mock(Response.class);
    when(okResponse.getStatusCode()).thenReturn(202);
    when(mockBatch.deleteBlob(anyString(), anyString())).thenReturn(okResponse);
    Observable<StoredProcedureResponse> mockBulkDeleteResponse = getMockBulkDeleteResponse(1);
    when(mockumentClient.executeStoredProcedure(anyString(), any(RequestOptions.class), any())).thenReturn(mockBulkDeleteResponse);
    String checkpointJson = objectMapper.writeValueAsString(AzureStorageCompactor.emptyCheckpoints);
    mockCheckpointDownload(true, checkpointJson);
}

Example 2 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessorTest method testQueryChangeFeedNormal.

/**
 * Test change feed query.
 */
@Test
public void testQueryChangeFeedNormal() throws Exception {
    Observable<FeedResponse<Document>> mockResponse = mock(Observable.class);
    List<Document> docList = Collections.singletonList(AzureTestUtils.createDocumentFromCloudBlobMetadata(blobMetadata));
    mockObservableForChangeFeedQuery(docList, mockResponse);
    when(mockumentClient.queryDocumentChangeFeed(anyString(), any(ChangeFeedOptions.class))).thenReturn(mockResponse);
    // test with non null requestContinuationToken
    List<CloudBlobMetadata> metadataList = doQueryChangeFeed("test");
    assertEquals("Expected single entry", 1, metadataList.size());
    CloudBlobMetadata outputMetadata = metadataList.get(0);
    assertEquals("Returned metadata does not match original", blobMetadata, outputMetadata);
    assertEquals(1, azureMetrics.changeFeedQueryCount.getCount());
    assertEquals(0, azureMetrics.changeFeedQueryFailureCount.getCount());
    // test with a null continuation token
    metadataList = doQueryChangeFeed(null);
    assertEquals("Expected single entry", 1, metadataList.size());
    outputMetadata = metadataList.get(0);
    assertEquals("Returned metadata does not match original", blobMetadata, outputMetadata);
    assertEquals(2, azureMetrics.changeFeedQueryCount.getCount());
    assertEquals(0, azureMetrics.changeFeedQueryFailureCount.getCount());
    // test when queryChangeFeed throws exception
    when(mockumentClient.queryDocumentChangeFeed(anyString(), any(ChangeFeedOptions.class))).thenThrow(new RuntimeException("mock exception", new DocumentClientException(404)));
    try {
        doQueryChangeFeed(null);
    } catch (DocumentClientException e) {
    }
    assertEquals(3, azureMetrics.changeFeedQueryCount.getCount());
    assertEquals(1, azureMetrics.changeFeedQueryFailureCount.getCount());
}

Example 3 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessor method getDeprecatedContainers.

/**
 * Fetch a {@link Set} of {@link CosmosContainerDeletionEntry} objects from cosmos db that are not marked as deleted.
 * @param maxEntries Max number of entries to fetch on one query.
 * @return {@link Set} of {@link CosmosContainerDeletionEntry} objects.
 * @throws DocumentClientException in case of any error.
 */
public Set<CosmosContainerDeletionEntry> getDeprecatedContainers(int maxEntries) throws DocumentClientException {
    SqlQuerySpec querySpec = new SqlQuerySpec(DEPRECATED_CONTAINERS_QUERY, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries)));
    Timer timer = new Timer();
    Set<CosmosContainerDeletionEntry> containerDeletionEntries = new HashSet<>();
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(cosmosDeletedContainerCollectionLink, null, querySpec, new FeedOptions(), timer).getIterator();
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            response.getResults().iterator().forEachRemaining(doc -> containerDeletionEntries.add(CosmosContainerDeletionEntry.fromJson(new JSONObject(doc.toJson()))));
        }
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Get deprecated containers query {} got {}", querySpec.getQueryText(), ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
    return containerDeletionEntries;
}

Example 4 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class CosmosDataAccessor method getDeadBlobs.

/**
 * Get the list of blobs in the specified partition that have been deleted or expired for at least the
 * configured retention period.
 * @param partitionPath the partition to query.
 * @param fieldName the field name to query on. Allowed values are {@link CloudBlobMetadata#FIELD_DELETION_TIME} and
 *                  {@link CloudBlobMetadata#FIELD_EXPIRATION_TIME}.
 * @param startTime the start of the query time range.
 * @param endTime the end of the query time range.
 * @param maxEntries the max number of metadata records to return.
 * @return a List of {@link CloudBlobMetadata} referencing the dead blobs found.
 * @throws DocumentClientException
 */
List<CloudBlobMetadata> getDeadBlobs(String partitionPath, String fieldName, long startTime, long endTime, int maxEntries) throws DocumentClientException {
    String deadBlobsQuery;
    if (fieldName.equals(CloudBlobMetadata.FIELD_DELETION_TIME)) {
        deadBlobsQuery = DELETED_BLOBS_QUERY;
    } else if (fieldName.equals(CloudBlobMetadata.FIELD_EXPIRATION_TIME)) {
        deadBlobsQuery = EXPIRED_BLOBS_QUERY;
    } else {
        throw new IllegalArgumentException("Invalid field: " + fieldName);
    }
    SqlQuerySpec querySpec = new SqlQuerySpec(deadBlobsQuery, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries), new SqlParameter(START_TIME_PARAM, startTime), new SqlParameter(END_TIME_PARAM, endTime)));
    FeedOptions feedOptions = new FeedOptions();
    feedOptions.setMaxItemCount(maxEntries);
    feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
    feedOptions.setPartitionKey(new PartitionKey(partitionPath));
    try {
        Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, azureMetrics.deadBlobsQueryTime).getIterator();
        List<CloudBlobMetadata> deadBlobsList = new ArrayList<>();
        double requestCharge = 0.0;
        while (iterator.hasNext()) {
            FeedResponse<Document> response = iterator.next();
            requestCharge += response.getRequestCharge();
            response.getResults().iterator().forEachRemaining(doc -> deadBlobsList.add(createMetadataFromDocument(doc)));
        }
        if (requestCharge >= requestChargeThreshold) {
            logger.info("Dead blobs query partition {} endTime {} request charge {} for {} records", partitionPath, new Date(endTime), requestCharge, deadBlobsList.size());
        }
        return deadBlobsList;
    } catch (RuntimeException rex) {
        if (rex.getCause() instanceof DocumentClientException) {
            logger.warn("Dead blobs query {} partition {} got {}", deadBlobsQuery, partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
            throw (DocumentClientException) rex.getCause();
        }
        throw rex;
    }
}

Example 5 with FeedResponse

use of com.microsoft.azure.cosmosdb.FeedResponse in project ambry by linkedin.

the class AzureCloudDestinationTest method testFindEntriesSinceWithUniqueUpdateTimes.

/**
 * Test findEntriesSince with all entries having unique updateTimes.
 * @throws Exception
 */
private void testFindEntriesSinceWithUniqueUpdateTimes(AzureCloudDestination azureDest) throws Exception {
    long chunkSize = 110000;
    // between 9 and 10 chunks
    long maxTotalSize = 1000000;
    long startTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
    int totalBlobs = 20;
    // create metadata list where total size > maxTotalSize
    List<Document> docList = new ArrayList<>();
    List<String> blobIdList = new ArrayList<>();
    for (int j = 0; j < totalBlobs; j++) {
        BlobId blobId = generateBlobId();
        blobIdList.add(blobId.getID());
        CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, creationTime, Utils.Infinite_Time, chunkSize, CloudBlobMetadata.EncryptionOrigin.NONE);
        inputMetadata.setUploadTime(startTime + j);
        docList.add(AzureTestUtils.createDocumentFromCloudBlobMetadata(inputMetadata, startTime + j));
    }
    Observable<FeedResponse<Document>> mockResponse = mock(Observable.class);
    mockObservableForQuery(docList, mockResponse);
    when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(mockResponse);
    CosmosUpdateTimeFindToken findToken = new CosmosUpdateTimeFindToken();
    // Run the query
    FindResult findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, maxTotalSize);
    List<CloudBlobMetadata> firstResult = findResult.getMetadataList();
    findToken = (CosmosUpdateTimeFindToken) findResult.getUpdatedFindToken();
    assertEquals("Did not get expected doc count", maxTotalSize / chunkSize, firstResult.size());
    docList = docList.subList(firstResult.size(), docList.size());
    assertEquals("Find token has wrong last update time", findToken.getLastUpdateTime(), firstResult.get(firstResult.size() - 1).getLastUpdateTime());
    assertEquals("Find token has wrong lastUpdateTimeReadBlobIds", findToken.getLastUpdateTimeReadBlobIds(), new HashSet<>(Collections.singletonList(firstResult.get(firstResult.size() - 1).getId())));
    mockObservableForQuery(docList, mockResponse);
    findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, maxTotalSize);
    List<CloudBlobMetadata> secondResult = findResult.getMetadataList();
    findToken = (CosmosUpdateTimeFindToken) findResult.getUpdatedFindToken();
    assertEquals("Unexpected doc count", maxTotalSize / chunkSize, secondResult.size());
    assertEquals("Unexpected first blobId", blobIdList.get(firstResult.size()), secondResult.get(0).getId());
    mockObservableForQuery(docList, mockResponse);
    findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, chunkSize / 2);
    // Rerun with max size below blob size, and make sure it returns one result
    assertEquals("Expected one result", 1, findResult.getMetadataList().size());
}