Examples with FindResult com.github.ambry.cloud.FindResult used on opensource projects

Example 6 with FindResult

use of com.github.ambry.cloud.FindResult in project ambry by linkedin.

the class AzureIntegrationTest method testFindEntriesSince.

/**
 * Test findEntriesSince with specified cloud token factory.
 * @param replicationCloudTokenFactory the factory to use.
 * @throws Exception on error
 */
private void testFindEntriesSince(String replicationCloudTokenFactory) throws Exception {
    logger.info("Testing findEntriesSince with {}", replicationCloudTokenFactory);
    testProperties.setProperty(ReplicationConfig.REPLICATION_CLOUD_TOKEN_FACTORY, replicationCloudTokenFactory);
    VerifiableProperties verifiableProperties = new VerifiableProperties(testProperties);
    ReplicationConfig replicationConfig = new ReplicationConfig(verifiableProperties);
    FindTokenFactory findTokenFactory = new FindTokenHelper(null, replicationConfig).getFindTokenFactoryFromReplicaType(ReplicaType.CLOUD_BACKED);
    azureDest = (AzureCloudDestination) new AzureCloudDestinationFactory(verifiableProperties, new MetricRegistry(), clusterMap).getCloudDestination();
    cleanup();
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    String partitionPath = String.valueOf(testPartition);
    // Upload some blobs with different upload times
    int blobCount = 90;
    int chunkSize = 1000;
    int maxTotalSize = 20000;
    int expectedNumQueries = (blobCount * chunkSize) / maxTotalSize + 1;
    long now = System.currentTimeMillis();
    long startTime = now - TimeUnit.DAYS.toMillis(7);
    for (int j = 0; j < blobCount; j++) {
        BlobId blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
        InputStream inputStream = getBlobInputStream(chunkSize);
        CloudBlobMetadata cloudBlobMetadata = new CloudBlobMetadata(blobId, startTime, Utils.Infinite_Time, chunkSize, CloudBlobMetadata.EncryptionOrigin.VCR, vcrKmsContext, cryptoAgentFactory, chunkSize, (short) 0);
        cloudBlobMetadata.setUploadTime(startTime + j * 1000);
        assertTrue("Expected upload to return true", uploadBlobWithRetry(blobId, chunkSize, cloudBlobMetadata, inputStream, cloudRequestAgent, azureDest));
    }
    FindToken findToken = findTokenFactory.getNewFindToken();
    // Call findEntriesSince in a loop until no new entries are returned
    FindResult findResult;
    int numQueries = 0;
    int totalBlobsReturned = 0;
    do {
        findResult = findEntriesSinceWithRetry(partitionPath, findToken, maxTotalSize);
        findToken = findResult.getUpdatedFindToken();
        if (!findResult.getMetadataList().isEmpty()) {
            numQueries++;
        }
        totalBlobsReturned += findResult.getMetadataList().size();
    } while (!noMoreFindSinceEntries(findResult, findToken));
    assertEquals("Wrong number of queries", expectedNumQueries, numQueries);
    assertEquals("Wrong number of blobs", blobCount, totalBlobsReturned);
    assertEquals("Wrong byte count", blobCount * chunkSize, findToken.getBytesRead());
    cleanup();
}

Example 7 with FindResult

use of com.github.ambry.cloud.FindResult in project ambry by linkedin.

the class CosmosUpdateTimeBasedReplicationFeed method getNextEntriesAndUpdatedToken.

@Override
public FindResult getNextEntriesAndUpdatedToken(FindToken curfindToken, long maxTotalSizeOfEntries, String partitionPath) throws DocumentClientException {
    Timer.Context operationTimer = azureMetrics.replicationFeedQueryTime.time();
    try {
        CosmosUpdateTimeFindToken findToken = (CosmosUpdateTimeFindToken) curfindToken;
        SqlQuerySpec entriesSinceQuery = new SqlQuerySpec(ENTRIES_SINCE_QUERY_TEMPLATE, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, queryBatchSize), new SqlParameter(TIME_SINCE_PARAM, findToken.getLastUpdateTime())));
        List<CloudBlobMetadata> queryResults = cosmosDataAccessor.queryMetadata(partitionPath, entriesSinceQuery, azureMetrics.findSinceQueryTime);
        if (queryResults.isEmpty()) {
            return new FindResult(new ArrayList<>(), findToken);
        }
        if (queryResults.get(0).getLastUpdateTime() == findToken.getLastUpdateTime()) {
            filterOutLastReadBlobs(queryResults, findToken.getLastUpdateTimeReadBlobIds(), findToken.getLastUpdateTime());
        }
        List<CloudBlobMetadata> cappedResults = CloudBlobMetadata.capMetadataListBySize(queryResults, maxTotalSizeOfEntries);
        return new FindResult(cappedResults, CosmosUpdateTimeFindToken.getUpdatedToken(findToken, cappedResults));
    } finally {
        operationTimer.stop();
    }
}