Examples with BlobId com.github.ambry.commons.BlobId used on opensource projects

Example 16 with BlobId

use of com.github.ambry.commons.BlobId in project ambry by linkedin.

the class AzureCloudDestinationTest method setup.

@Before
public void setup() throws Exception {
    long partition = 666;
    PartitionId partitionId = new MockPartitionId(partition, MockClusterMap.DEFAULT_PARTITION_CLASS);
    blobId = new BlobId(BLOB_ID_V6, BlobIdType.NATIVE, dataCenterId, accountId, containerId, partitionId, false, BlobDataType.DATACHUNK);
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, 0, Utils.Infinite_Time, blobSize, CloudBlobMetadata.EncryptionOrigin.NONE);
    mockServiceClient = mock(BlobServiceClient.class);
    mockBlobBatchClient = mock(BlobBatchClient.class);
    mockBlockBlobClient = AzureBlobDataAccessorTest.setupMockBlobClient(mockServiceClient);
    mockBlobExistence(false);
    mockumentClient = mock(AsyncDocumentClient.class);
    Observable<ResourceResponse<Document>> mockResponse = getMockedObservableForSingleResource(blobMetadata);
    when(mockumentClient.readDocument(anyString(), any(RequestOptions.class))).thenReturn(mockResponse);
    when(mockumentClient.upsertDocument(anyString(), any(Object.class), any(RequestOptions.class), anyBoolean())).thenReturn(mockResponse);
    when(mockumentClient.replaceDocument(any(Document.class), any(RequestOptions.class))).thenReturn(mockResponse);
    when(mockumentClient.deleteDocument(anyString(), any(RequestOptions.class))).thenReturn(mockResponse);
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_CONNECTION_STRING, storageConnection);
    configProps.setProperty(AzureCloudConfig.COSMOS_ENDPOINT, "http://ambry.beyond-the-cosmos.com:443");
    configProps.setProperty(AzureCloudConfig.COSMOS_COLLECTION_LINK, "ambry/metadata");
    configProps.setProperty(AzureCloudConfig.COSMOS_DELETED_CONTAINER_COLLECTION_LINK, "ambry/deletedContainer");
    configProps.setProperty(AzureCloudConfig.COSMOS_KEY, "cosmos-key");
    configProps.setProperty("clustermap.cluster.name", "main");
    configProps.setProperty("clustermap.datacenter.name", "uswest");
    configProps.setProperty("clustermap.host.name", "localhost");
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_AUTHORITY, "https://login.microsoftonline.com/test-account/");
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_CLIENTID, "client-id");
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_SECRET, "client-secret");
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_ENDPOINT, "https://azure_storage.blob.core.windows.net");
    configProps.setProperty(AzureCloudConfig.AZURE_STORAGE_CLIENT_CLASS, "com.github.ambry.cloud.azure.ConnectionStringBasedStorageClient");
    vcrMetrics = new VcrMetrics(new MetricRegistry());
    azureMetrics = new AzureMetrics(new MetricRegistry());
    clusterMap = mock(ClusterMap.class);
    azureDest = new AzureCloudDestination(mockServiceClient, mockBlobBatchClient, mockumentClient, "foo", "bar", clusterName, azureMetrics, defaultAzureReplicationFeedType, clusterMap, false, configProps);
}

Example 17 with BlobId

use of com.github.ambry.commons.BlobId in project ambry by linkedin.

the class CloudBlobStoreTest method testStoreDeletes.

/**
 * Test the CloudBlobStore delete method.
 */
@Test
public void testStoreDeletes() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    int count = 10;
    long now = System.currentTimeMillis();
    Map<BlobId, MessageInfo> messageInfoMap = new HashMap<>();
    for (int j = 0; j < count; j++) {
        BlobId blobId = getUniqueId(refAccountId, refContainerId, true, partitionId);
        messageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, initLifeVersion()));
    }
    store.delete(new ArrayList<>(messageInfoMap.values()));
    verify(dest, times(count)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count, 0);
    } else {
        verifyCacheHits(0, 0);
    }
    // If not isVcr, deletion should fail
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    int expectedCount = isVcr ? count : count * 2;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 2, count);
    } else {
        verifyCacheHits(0, 0);
    }
    // Call again with a smaller life version. If isVcr, should hit the cache again.
    Map<BlobId, MessageInfo> newMessageInfoMap = new HashMap<>();
    for (BlobId blobId : messageInfoMap.keySet()) {
        newMessageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, initLifeVersion()));
    }
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    expectedCount = isVcr ? count : count * 3;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 3, count * 2);
    } else {
        verifyCacheHits(0, 0);
    }
    // Call again with a larger life version. Should not hit cache again.
    for (BlobId blobId : messageInfoMap.keySet()) {
        newMessageInfoMap.put(blobId, new MessageInfo(blobId, SMALL_BLOB_SIZE, refAccountId, refContainerId, now, (short) 2));
    }
    for (MessageInfo messageInfo : messageInfoMap.values()) {
        try {
            store.delete(Collections.singletonList(messageInfo));
        } catch (StoreException ex) {
            if (isVcr) {
                assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
            }
        }
    }
    expectedCount = isVcr ? count : count * 4;
    verify(dest, times(expectedCount)).deleteBlob(any(BlobId.class), eq(now), anyShort(), any(CloudUpdateValidator.class));
    if (isVcr) {
        verifyCacheHits(count * 4, count * 3);
    } else {
        verifyCacheHits(0, 0);
    }
    // Try to upload a set of blobs containing duplicates
    List<MessageInfo> messageInfoList = new ArrayList<>(messageInfoMap.values());
    messageInfoList.add(messageInfoList.get(messageInfoMap.values().size() - 1));
    try {
        store.delete(messageInfoList);
    } catch (IllegalArgumentException iaex) {
    }
    if (isVcr) {
        verifyCacheHits(count * 4, count * 3);
    } else {
        verifyCacheHits(0, 0);
    }
}

Example 18 with BlobId

use of com.github.ambry.commons.BlobId in project ambry by linkedin.

the class CloudBlobStoreTest method testExceptionRetry.

/* Test retry behavior */
@Test
public void testExceptionRetry() throws Exception {
    CloudDestination exDest = mock(CloudDestination.class);
    long retryDelay = 5;
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    BlobId blobId = CloudTestUtil.addBlobToMessageSet(messageWriteSet, SMALL_BLOB_SIZE, Utils.Infinite_Time, refAccountId, refContainerId, false, false, partitionId, operationTime, isVcr);
    List<StoreKey> keys = Collections.singletonList(blobId);
    CloudBlobMetadata metadata = new CloudBlobMetadata(blobId, operationTime, Utils.Infinite_Time, 1024, null);
    CloudStorageException retryableException = new CloudStorageException("Server unavailable", null, 500, true, retryDelay);
    when(exDest.uploadBlob(any(BlobId.class), anyLong(), any(), any(InputStream.class))).thenAnswer((invocation -> {
        consumeStream(invocation);
        throw retryableException;
    })).thenAnswer(invocation -> {
        consumeStream(invocation);
        return true;
    });
    when(exDest.deleteBlob(any(BlobId.class), anyLong(), anyShort(), any(CloudUpdateValidator.class))).thenThrow(retryableException).thenReturn(true);
    when(exDest.updateBlobExpiration(any(BlobId.class), anyLong(), any(CloudUpdateValidator.class))).thenThrow(retryableException).thenReturn((short) 1);
    when(exDest.getBlobMetadata(anyList())).thenThrow(retryableException).thenReturn(Collections.singletonMap(metadata.getId(), metadata));
    doThrow(retryableException).doNothing().when(exDest).downloadBlob(any(BlobId.class), any());
    Properties props = new Properties();
    setBasicProperties(props);
    props.setProperty(CloudConfig.VCR_REQUIRE_ENCRYPTION, "false");
    props.setProperty(CloudConfig.CLOUD_BLOB_CRYPTO_AGENT_FACTORY_CLASS, TestCloudBlobCryptoAgentFactory.class.getName());
    vcrMetrics = new VcrMetrics(new MetricRegistry());
    CloudBlobStore exStore = spy(new CloudBlobStore(new VerifiableProperties(props), partitionId, exDest, clusterMap, vcrMetrics));
    exStore.start();
    // Run all operations, they should be retried and succeed second time.
    int expectedCacheLookups = 0, expectedCacheRemoves = 0, expectedRetries = 0;
    // PUT
    exStore.put(messageWriteSet);
    expectedRetries++;
    expectedCacheLookups++;
    // UPDATE_TTL
    exStore.updateTtl(messageWriteSet.getMessageSetInfo());
    expectedRetries++;
    expectedCacheRemoves++;
    expectedCacheLookups += 2;
    // DELETE
    exStore.delete(messageWriteSet.getMessageSetInfo());
    expectedRetries++;
    expectedCacheRemoves++;
    if (isVcr) {
        // no cache lookup in case of delete for frontend.
        expectedCacheLookups += 2;
    }
    // GET
    exStore.get(keys, EnumSet.noneOf(StoreGetOptions.class));
    expectedRetries++;
    exStore.downloadBlob(metadata, blobId, new ByteArrayOutputStream());
    expectedRetries++;
    // Expect retries for all ops, cache lookups for the first three
    assertEquals("Unexpected retry count", expectedRetries, vcrMetrics.retryCount.getCount());
    assertEquals("Unexpected wait time", expectedRetries * retryDelay, vcrMetrics.retryWaitTimeMsec.getCount());
    verifyCacheHits(expectedCacheLookups, 0);
    verify(exStore, times(expectedCacheRemoves)).removeFromCache(eq(blobId.getID()));
    // Rerun the first three, should all skip due to cache hit
    messageWriteSet.resetBuffers();
    int expectedCacheHits = 0;
    try {
        exStore.put(messageWriteSet);
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.Already_Exist);
    }
    expectedCacheHits++;
    exStore.addToCache(blobId.getID(), (short) 0, CloudBlobStore.BlobState.TTL_UPDATED);
    exStore.updateTtl(messageWriteSet.getMessageSetInfo());
    expectedCacheHits++;
    exStore.addToCache(blobId.getID(), (short) 0, CloudBlobStore.BlobState.DELETED);
    try {
        exStore.delete(messageWriteSet.getMessageSetInfo());
    } catch (StoreException ex) {
        assertEquals(ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    if (isVcr) {
        expectedCacheHits++;
    }
    verifyCacheHits(expectedCacheLookups + expectedCacheHits, expectedCacheHits);
}

Example 19 with BlobId

use of com.github.ambry.commons.BlobId in project ambry by linkedin.

the class CloudBlobStoreTest method testFindMissingKeys.

/**
 * Test the CloudBlobStore findMissingKeys method.
 */
@Test
public void testFindMissingKeys() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    int count = 10;
    List<StoreKey> keys = new ArrayList<>();
    Map<String, CloudBlobMetadata> metadataMap = new HashMap<>();
    for (int j = 0; j < count; j++) {
        // Blob with metadata
        BlobId existentBlobId = getUniqueId(refAccountId, refContainerId, false, partitionId);
        keys.add(existentBlobId);
        metadataMap.put(existentBlobId.getID(), new CloudBlobMetadata(existentBlobId, operationTime, Utils.Infinite_Time, 1024, CloudBlobMetadata.EncryptionOrigin.ROUTER));
        // Blob without metadata
        BlobId nonexistentBlobId = getUniqueId(refAccountId, refContainerId, false, partitionId);
        keys.add(nonexistentBlobId);
    }
    when(dest.getBlobMetadata(anyList())).thenReturn(metadataMap);
    Set<StoreKey> missingKeys = store.findMissingKeys(keys);
    verify(dest).getBlobMetadata(anyList());
    int expectedLookups = keys.size();
    int expectedHits = 0;
    verifyCacheHits(expectedLookups, expectedHits);
    assertEquals("Wrong number of missing keys", count, missingKeys.size());
    if (isVcr) {
        // Add keys to cache and rerun (should be cached)
        for (StoreKey storeKey : keys) {
            store.addToCache(storeKey.getID(), (short) 0, CloudBlobStore.BlobState.CREATED);
        }
        missingKeys = store.findMissingKeys(keys);
        assertTrue("Expected no missing keys", missingKeys.isEmpty());
        expectedLookups += keys.size();
        expectedHits += keys.size();
        verifyCacheHits(expectedLookups, expectedHits);
        // getBlobMetadata should not have been called a second time.
        verify(dest).getBlobMetadata(anyList());
    }
}

Example 20 with BlobId

use of com.github.ambry.commons.BlobId in project ambry by linkedin.

the class CloudTestUtil method addBlobToMessageSet.

/**
 * Utility method to generate a BlobId and byte buffer for a blob with specified properties and add them to the specified MessageWriteSet.
 * @param messageWriteSet the {@link MockMessageWriteSet} in which to store the data.
 * @param size the size of the byte buffer.
 * @param expiresAtMs the expiration time.
 * @param accountId the account Id.
 * @param containerId the container Id.
 * @param encrypted the encrypted bit.
 * @param deleted true if blob is deleted.
 * @param partitionId the partition id.
 * @param operationTime the operation time.
 * @param isVcr flag to indicate if running as vcr.
 * @return the generated {@link BlobId}.
 */
static BlobId addBlobToMessageSet(MockMessageWriteSet messageWriteSet, long size, long expiresAtMs, short accountId, short containerId, boolean encrypted, boolean deleted, PartitionId partitionId, long operationTime, boolean isVcr) {
    BlobId id = getUniqueId(accountId, containerId, encrypted, partitionId);
    long crc = new Random().nextLong();
    MessageInfo info = new MessageInfo(id, size, deleted, true, false, expiresAtMs, crc, accountId, containerId, operationTime, initLifeVersion(isVcr));
    ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes((int) size));
    messageWriteSet.add(info, buffer);
    return id;
}