Examples with CosmosChangeFeedFindToken com.github.ambry.cloud.azure.CosmosChangeFeedFindToken used on opensource projects

Example 1 with CosmosChangeFeedFindToken

use of com.github.ambry.cloud.azure.CosmosChangeFeedFindToken in project ambry by linkedin.

the class CloudBlobStoreTest method testFindEntriesSince.

/**
 * Test the CloudBlobStore findEntriesSince method.
 */
@Test
public void testFindEntriesSince() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    long maxTotalSize = 1000000;
    // 1) start with empty token, call find, return some data
    long blobSize = 200000;
    int numBlobsFound = 5;
    CosmosChangeFeedFindToken cosmosChangeFeedFindToken = new CosmosChangeFeedFindToken(blobSize * numBlobsFound, "start", "end", 0, numBlobsFound, UUID.randomUUID().toString());
    // create a list of 10 blobs with total size less than maxSize, and return it as part of query ChangeFeed
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), cosmosChangeFeedFindToken));
    CosmosChangeFeedFindToken startToken = new CosmosChangeFeedFindToken();
    // remote node host name and replica path are not really used by cloud store, it's fine to keep them null
    FindInfo findInfo = store.findEntriesSince(startToken, maxTotalSize, null, null);
    CosmosChangeFeedFindToken outputToken = (CosmosChangeFeedFindToken) findInfo.getFindToken();
    assertEquals(blobSize * numBlobsFound, outputToken.getBytesRead());
    assertEquals(numBlobsFound, outputToken.getTotalItems());
    assertEquals(0, outputToken.getIndex());
    // 2) call find with new token, return more data including lastBlob, verify token updated
    cosmosChangeFeedFindToken = new CosmosChangeFeedFindToken(blobSize * 2 * numBlobsFound, "start2", "end2", 0, numBlobsFound, UUID.randomUUID().toString());
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), cosmosChangeFeedFindToken));
    findInfo = store.findEntriesSince(outputToken, maxTotalSize, null, null);
    outputToken = (CosmosChangeFeedFindToken) findInfo.getFindToken();
    assertEquals(blobSize * 2 * numBlobsFound, outputToken.getBytesRead());
    assertEquals(numBlobsFound, outputToken.getTotalItems());
    assertEquals(0, outputToken.getIndex());
    // 3) call find with new token, no more data, verify token unchanged
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), outputToken));
    findInfo = store.findEntriesSince(outputToken, maxTotalSize, null, null);
    assertTrue(findInfo.getMessageEntries().isEmpty());
    FindToken finalToken = findInfo.getFindToken();
    assertEquals(outputToken, finalToken);
}

Example 2 with CosmosChangeFeedFindToken

use of com.github.ambry.cloud.azure.CosmosChangeFeedFindToken in project ambry by linkedin.

the class LatchBasedInMemoryCloudDestination method findChangeFeedBasedEntries.

/**
 * Populates an ordered sequenced list of blobs in the specified partition in {@code nextEntries} {@link List},
 * ordered by change feed. Returns the updated {@link com.github.ambry.replication.FindToken}.
 * @param partitionPath the partition to query.
 * @param findToken the {@link com.github.ambry.replication.FindToken} specifying the boundary for the query.
 * @param maxTotalSizeOfEntries the cumulative size limit for the list of blobs returned.
 * @return {@link FindResult} instance that contains updated {@link CosmosChangeFeedFindToken} object which can act as a bookmark for
 * subsequent requests, and {@link List} of {@link CloudBlobMetadata} entries.
 * @throws CloudStorageException
 */
private FindResult findChangeFeedBasedEntries(String partitionPath, FindToken findToken, long maxTotalSizeOfEntries) {
    List<CloudBlobMetadata> nextEntries = new ArrayList<>();
    String continuationToken = ((CosmosChangeFeedFindToken) findToken).getEndContinuationToken();
    List<BlobId> blobIds = new ArrayList<>();
    getFeed(continuationToken, maxTotalSizeOfEntries, blobIds);
    nextEntries.addAll(blobIds.stream().map(blobId -> map.get(blobId).getFirst()).collect(Collectors.toList()));
    CosmosChangeFeedFindToken cosmosChangeFeedFindToken = (CosmosChangeFeedFindToken) findToken;
    if (blobIds.size() != 0) {
        long bytesToBeRead = nextEntries.stream().mapToLong(CloudBlobMetadata::getSize).sum();
        cosmosChangeFeedFindToken = new CosmosChangeFeedFindToken(bytesToBeRead, changeFeed.getContinuationTokenForBlob(blobIds.get(0)), createEndContinuationToken(blobIds), 0, blobIds.size(), changeFeed.getReqUuid(), cosmosChangeFeedFindToken.getVersion());
    }
    return new FindResult(nextEntries, cosmosChangeFeedFindToken);
}