Examples with CloudBlobMetadata com.github.ambry.cloud.CloudBlobMetadata used on opensource projects

Example 41 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureCloudDestinationTest method testFindEntriesSinceUsingChangeFeed.

/**
 * Test findEntriesSince when cloud destination uses change feed based token.
 */
@Test
public void testFindEntriesSinceUsingChangeFeed() throws Exception {
    long chunkSize = 110000;
    // between 9 and 10 chunks
    long maxTotalSize = 1000000;
    long startTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
    int totalBlobs = 20;
    // create metadata list where total size > maxTotalSize
    List<String> blobIdList = new ArrayList<>();
    List<CloudBlobMetadata> cloudBlobMetadataList = new ArrayList<>();
    for (int j = 0; j < totalBlobs; j++) {
        BlobId blobId = generateBlobId();
        blobIdList.add(blobId.getID());
        CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, creationTime, Utils.Infinite_Time, chunkSize, CloudBlobMetadata.EncryptionOrigin.NONE);
        inputMetadata.setUploadTime(startTime + j);
        cloudBlobMetadataList.add(inputMetadata);
    }
    MockChangeFeedQuery mockChangeFeedQuery = new MockChangeFeedQuery();
    AzureReplicationFeed azureReplicationFeed = null;
    try {
        azureReplicationFeed = new CosmosChangeFeedBasedReplicationFeed(mockChangeFeedQuery, azureMetrics, azureDest.getQueryBatchSize());
        FieldSetter.setField(azureDest, azureDest.getClass().getDeclaredField("azureReplicationFeed"), azureReplicationFeed);
        cloudBlobMetadataList.stream().forEach(doc -> mockChangeFeedQuery.add(doc));
        CosmosChangeFeedFindToken findToken = new CosmosChangeFeedFindToken();
        // Run the query
        FindResult findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, maxTotalSize);
        List<CloudBlobMetadata> firstResult = findResult.getMetadataList();
        findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
        assertEquals("Did not get expected doc count", maxTotalSize / chunkSize, firstResult.size());
        assertEquals("Find token has wrong end continuation token", (findToken).getIndex(), firstResult.size());
        assertEquals("Find token has wrong totalItems count", (findToken).getTotalItems(), Math.min(blobIdList.size(), azureDest.getQueryBatchSize()));
        assertEquals("Unexpected change feed cache miss count", 1, azureMetrics.changeFeedCacheMissRate.getCount());
        assertEquals("Unexpected change feed cache refresh count", 0, azureMetrics.changeFeedCacheRefreshRate.getCount());
        assertEquals("Unexpected change feed cache hit count", 0, azureMetrics.changeFeedCacheHitRate.getCount());
        cloudBlobMetadataList = cloudBlobMetadataList.subList(firstResult.size(), cloudBlobMetadataList.size());
        findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, maxTotalSize);
        List<CloudBlobMetadata> secondResult = findResult.getMetadataList();
        findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
        assertEquals("Unexpected doc count", maxTotalSize / chunkSize, secondResult.size());
        assertEquals("Unexpected first blobId", blobIdList.get(firstResult.size()), secondResult.get(0).getId());
        assertEquals("Find token has wrong totalItems count", (findToken).getTotalItems(), Math.min(blobIdList.size(), azureDest.getQueryBatchSize()));
        assertEquals("Unexpected change feed cache miss count", 1, azureMetrics.changeFeedCacheMissRate.getCount());
        assertEquals("Unexpected change feed cache refresh count", 0, azureMetrics.changeFeedCacheRefreshRate.getCount());
        assertEquals("Unexpected change feed cache hit count", 1, azureMetrics.changeFeedCacheHitRate.getCount());
        // Rerun with max size below blob size, and make sure it returns one result
        findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, chunkSize - 1);
        List<CloudBlobMetadata> thirdResult = findResult.getMetadataList();
        assertEquals("Expected one result", 1, thirdResult.size());
        findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
        assertEquals("Unexpected change feed cache miss count", 1, azureMetrics.changeFeedCacheMissRate.getCount());
        assertEquals("Unexpected change feed cache refresh count", 0, azureMetrics.changeFeedCacheRefreshRate.getCount());
        assertEquals("Unexpected change feed cache hit count", 2, azureMetrics.changeFeedCacheHitRate.getCount());
        // Add more than AzureCloudConfig.cosmosQueryBatchSize blobs and test for correct change feed cache hits and misses.
        AzureCloudConfig azureConfig = new AzureCloudConfig(new VerifiableProperties(configProps));
        for (int j = 0; j < azureConfig.cosmosQueryBatchSize + 5; j++) {
            BlobId blobId = generateBlobId();
            blobIdList.add(blobId.getID());
            CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, creationTime, Utils.Infinite_Time, 10, CloudBlobMetadata.EncryptionOrigin.NONE);
            inputMetadata.setUploadTime(startTime + j);
            cloudBlobMetadataList.add(inputMetadata);
        }
        cloudBlobMetadataList.stream().forEach(doc -> mockChangeFeedQuery.add(doc));
        // Final correct query to drain out all the blobs and trigger a cache refresh.
        String prevEndToken = findToken.getEndContinuationToken();
        findResult = azureDest.findEntriesSince(blobId.getPartition().toPathString(), findToken, 1000000);
        findToken = (CosmosChangeFeedFindToken) findResult.getUpdatedFindToken();
        assertEquals("Unexpected change feed cache miss count", 1, azureMetrics.changeFeedCacheMissRate.getCount());
        assertEquals("Unexpected change feed cache refresh count", 1, azureMetrics.changeFeedCacheRefreshRate.getCount());
        assertEquals("Unexpected change feed cache hit count", 3, azureMetrics.changeFeedCacheHitRate.getCount());
        assertEquals("Since this would have triggered refresh, start token should have been previous token's end token", prevEndToken, findToken.getStartContinuationToken());
        // Query changefeed with invalid token and check for cache miss
        testFindEntriesSinceUsingChangeFeedWithInvalidToken(findToken);
    } finally {
        if (azureReplicationFeed != null) {
            azureReplicationFeed.close();
        }
    }
}

Example 42 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureCloudDestinationTest method testGetOneMetadata.

/**
 * Test to make sure that getting metadata for single blob calls ABS when not vcr and Cosmos when vcr.
 */
@Test
public void testGetOneMetadata() throws Exception {
    // 
    // Test 1: isVcr = false (already setup)
    // 
    // Get for existing blob
    Response<BlobProperties> mockResponse = mock(Response.class);
    BlobProperties mockProperties = mock(BlobProperties.class);
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, 0, -1, 0, null);
    Map<String, String> propertyMap = blobMetadata.toMap();
    when(mockProperties.getMetadata()).thenReturn(propertyMap);
    when(mockResponse.getValue()).thenReturn(mockProperties);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenReturn(mockResponse);
    List<BlobId> singleBlobList = Collections.singletonList(blobId);
    Map<String, CloudBlobMetadata> metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertEquals("Expected map of one", 1, metadataMap.size());
    verify(mockBlockBlobClient).getPropertiesWithResponse(any(), any(), any());
    verifyZeroInteractions(mockumentClient);
    // Get for nonexistent blob
    BlobStorageException ex = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenThrow(ex);
    metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertTrue("Expected empty map", metadataMap.isEmpty());
    verify(mockBlockBlobClient, times(2)).getPropertiesWithResponse(any(), any(), any());
    verifyZeroInteractions(mockumentClient);
    // 
    // Test 2: isVcr = true
    // 
    azureDest.close();
    azureDest = new AzureCloudDestination(mockServiceClient, mockBlobBatchClient, mockumentClient, "foo", "bar", clusterName, azureMetrics, defaultAzureReplicationFeedType, clusterMap, true, configProps);
    // Existing blob
    List<Document> docList = Collections.singletonList(createDocumentFromCloudBlobMetadata(blobMetadata));
    Observable<FeedResponse<Document>> feedResponse = mock(Observable.class);
    mockObservableForQuery(docList, feedResponse);
    when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(feedResponse);
    metadataMap = azureDest.getBlobMetadata(singleBlobList);
    assertEquals("Expected map of one", 1, metadataMap.size());
    verify(mockumentClient).queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class));
    verify(mockBlockBlobClient, times(2)).getPropertiesWithResponse(any(), any(), any());
}

Example 43 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureCloudDestinationTest method testQueryMetadata.

/**
 * Test metadata query for different input count.
 * @param numBlobs the number of blobs to query.
 * @param expectedQueries the number of internal queries made after batching.
 * @throws Exception
 */
private void testQueryMetadata(int numBlobs, int expectedQueries) throws Exception {
    // Reset metrics
    azureMetrics = new AzureMetrics(new MetricRegistry());
    try {
        azureDest = new AzureCloudDestination(mockServiceClient, mockBlobBatchClient, mockumentClient, "foo", "bar", clusterName, azureMetrics, defaultAzureReplicationFeedType, clusterMap, false, configProps);
        List<BlobId> blobIdList = new ArrayList<>();
        List<Document> docList = new ArrayList<>();
        for (int j = 0; j < numBlobs; j++) {
            BlobId blobId = generateBlobId();
            blobIdList.add(blobId);
            CloudBlobMetadata inputMetadata = new CloudBlobMetadata(blobId, creationTime, Utils.Infinite_Time, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
            docList.add(AzureTestUtils.createDocumentFromCloudBlobMetadata(inputMetadata));
        }
        Observable<FeedResponse<Document>> mockResponse = mock(Observable.class);
        mockObservableForQuery(docList, mockResponse);
        when(mockumentClient.queryDocuments(anyString(), any(SqlQuerySpec.class), any(FeedOptions.class))).thenReturn(mockResponse);
        Set<BlobId> blobIdSet = new HashSet<>(blobIdList);
        assertEquals(blobIdList.size(), blobIdSet.size());
        Map<String, CloudBlobMetadata> metadataMap = azureDest.getBlobMetadata(blobIdList);
        for (BlobId blobId : blobIdList) {
            assertEquals("Unexpected id in metadata", blobId.getID(), metadataMap.get(blobId.getID()).getId());
        }
        assertEquals(expectedQueries, azureMetrics.documentQueryCount.getCount());
        assertEquals(expectedQueries, azureMetrics.missingKeysQueryTime.getCount());
    } finally {
        azureDest.close();
    }
}

Example 44 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureCloudDestinationTest method testDeleteAfterPartialCompaction.

/**
 * Test delete when record is in Cosmos but not ABS.
 */
@Test
public void testDeleteAfterPartialCompaction() throws Exception {
    BlobStorageException ex = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
    when(mockBlockBlobClient.getPropertiesWithResponse(any(), any(), any())).thenThrow(ex);
    // Rig Cosmos to return us a deleted blob on read request
    long deletionTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10);
    CloudBlobMetadata deletedMetadata = new CloudBlobMetadata().setId(blobId.getID()).setDeletionTime(deletionTime);
    Observable<ResourceResponse<Document>> mockResponse = getMockedObservableForSingleResource(deletedMetadata);
    when(mockumentClient.readDocument(anyString(), any(RequestOptions.class))).thenReturn(mockResponse);
    // Now delete the puppy, Cosmos record should get purged.
    try {
        assertFalse("Expected update to recover and return false", azureDest.deleteBlob(blobId, deletionTime, (short) 0, dummyCloudUpdateValidator));
    } catch (CloudStorageException cex) {
        assertTrue(cex.getCause() instanceof BlobStorageException);
        assertEquals(((BlobStorageException) cex.getCause()).getErrorCode(), BlobErrorCode.BLOB_NOT_FOUND);
    }
    assertEquals("Expected recovery", 1, azureMetrics.blobUpdateRecoverCount.getCount());
    verify(mockumentClient).deleteDocument(anyString(), any());
}

Example 45 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureIntegrationTest method testBatchQuery.

/**
 * Test batch query on large number of blobs.
 * @throws Exception on error
 */
@Test
public void testBatchQuery() throws Exception {
    cleanup();
    int numBlobs = 100;
    PartitionId partitionId = new MockPartitionId(testPartition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    long creationTime = System.currentTimeMillis();
    Map<BlobId, byte[]> blobIdtoDataMap = createUnencryptedPermanentBlobs(numBlobs, dataCenterId, accountId, containerId, partitionId, blobSize, cloudRequestAgent, azureDest, creationTime);
    List<BlobId> blobIdList = new ArrayList<>(blobIdtoDataMap.keySet());
    long uploadTime = System.currentTimeMillis() - creationTime;
    logger.info("Uploaded {} blobs in {} ms", numBlobs, uploadTime);
    Map<String, CloudBlobMetadata> metadataMap = getBlobMetadataWithRetry(blobIdList, partitionId.toPathString(), cloudRequestAgent, azureDest);
    assertEquals("Unexpected size of returned metadata map", numBlobs, metadataMap.size());
    for (BlobId blobId : blobIdList) {
        CloudBlobMetadata metadata = metadataMap.get(blobId.getID());
        assertNotNull("No metadata found for blobId: " + blobId, metadata);
        assertEquals("Unexpected metadata id", blobId.getID(), metadata.getId());
        assertEquals("Unexpected metadata accountId", accountId, metadata.getAccountId());
        assertEquals("Unexpected metadata containerId", containerId, metadata.getContainerId());
        assertEquals("Unexpected metadata partitionId", partitionId.toPathString(), metadata.getPartitionId());
        assertEquals("Unexpected metadata creationTime", creationTime, metadata.getCreationTime());
        assertEquals("Unexpected metadata encryption origin", CloudBlobMetadata.EncryptionOrigin.NONE, metadata.getEncryptionOrigin());
        verifyDownloadMatches(blobId, blobIdtoDataMap.get(blobId));
    }
    cleanup();
}