Examples with Container com.github.ambry.account.Container used on opensource projects

Example 41 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class NonBlockingRouterTest method testCompositeBlobDataChunksDeleteMaxDeleteOperation.

protected void testCompositeBlobDataChunksDeleteMaxDeleteOperation(int maxDeleteOperation) throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        if (maxDeleteOperation != 0) {
            props.setProperty(RouterConfig.ROUTER_BACKGROUND_DELETER_MAX_CONCURRENT_OPERATIONS, Integer.toString(maxDeleteOperation));
        }
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // metadata blob + data chunks.
        final AtomicReference<CountDownLatch> deletesDoneLatch = new AtomicReference<>();
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.get().countDown();
            }
        };
        NonBlockingRouterMetrics localMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
        router = new NonBlockingRouter(routerConfig, localMetrics, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        String deleteServiceId = "delete-service";
        Set<String> blobsToBeDeleted = getBlobsInServers(mockServerLayout);
        int getRequestCount = mockServerLayout.getCount(RequestOrResponseType.GetRequest);
        // The third iteration is to test the case where the blob has expired.
        for (int i = 0; i < 3; i++) {
            if (i == 2) {
                // Create a clean cluster and put another blob that immediate expires.
                setOperationParams();
                putBlobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
                blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
                Set<String> allBlobsInServer = getBlobsInServers(mockServerLayout);
                allBlobsInServer.removeAll(blobsToBeDeleted);
                blobsToBeDeleted = allBlobsInServer;
            }
            blobsThatAreDeleted.clear();
            deletesDoneLatch.set(new CountDownLatch(5));
            router.deleteBlob(blobId, deleteServiceId).get();
            Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
            Assert.assertTrue("All blobs in server are deleted", blobsThatAreDeleted.keySet().containsAll(blobsToBeDeleted));
            Assert.assertTrue("Only blobs in server are deleted", blobsToBeDeleted.containsAll(blobsThatAreDeleted.keySet()));
            for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
                String expectedServiceId = blobIdAndServiceId.getKey().equals(blobId) ? deleteServiceId : BackgroundDeleteRequest.SERVICE_ID_PREFIX + deleteServiceId;
                Assert.assertEquals("Unexpected service ID for deleted blob", expectedServiceId, blobIdAndServiceId.getValue());
            }
            // For 1 chunk deletion attempt, 1 background operation for Get is initiated which results in 2 Get Requests at
            // the servers.
            getRequestCount += 2;
            Assert.assertEquals("Only one attempt of chunk deletion should have been done", getRequestCount, mockServerLayout.getCount(RequestOrResponseType.GetRequest));
        }
        deletesDoneLatch.set(new CountDownLatch(5));
        router.deleteBlob(blobId, null).get();
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        Assert.assertEquals("Get should NOT have been skipped", 0, localMetrics.skippedGetBlobCount.getCount());
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 42 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class NonBlockingRouterTest method testSimpleBlobDelete.

/**
 * Test to ensure that for simple blob deletions, no additional background delete operations
 * are initiated.
 */
@Test
public void testSimpleBlobDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE;
        String deleteServiceId = "delete-service";
        // metadata blob + data chunks.
        final AtomicInteger deletesInitiated = new AtomicInteger();
        final AtomicReference<String> receivedDeleteServiceId = new AtomicReference<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                deletesInitiated.incrementAndGet();
                receivedDeleteServiceId.set(serviceId);
            }
        };
        Properties props = getNonBlockingRouterProperties("DC1");
        setRouter(props, new MockServerLayout(mockClusterMap), deleteTrackingNotificationSystem);
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        router.deleteBlob(blobId, deleteServiceId).get();
        long waitStart = SystemTime.getInstance().milliseconds();
        while (router.getBackgroundOperationsCount() != 0 && SystemTime.getInstance().milliseconds() < waitStart + AWAIT_TIMEOUT_MS) {
            Thread.sleep(1000);
        }
        Assert.assertEquals("All background operations should be complete ", 0, router.getBackgroundOperationsCount());
        Assert.assertEquals("Only the original blob deletion should have been initiated", 1, deletesInitiated.get());
        Assert.assertEquals("The delete service ID should match the expected value", deleteServiceId, receivedDeleteServiceId.get());
        Assert.assertEquals("Get should have been skipped", 1, routerMetrics.skippedGetBlobCount.getCount());
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 43 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class NonBlockingRouterTest method testUnsuccessfulPutDataChunkDelete.

/**
 * Test that if a composite blob put fails, the successfully put data chunks are deleted.
 */
@Test
public void testUnsuccessfulPutDataChunkDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // fails, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(2);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        // There are 4 chunks for this blob.
        // All put operations make one request to each local server as there are 3 servers overall in the local DC.
        // Set the state of the mock servers so that they return success for the first 2 requests in order to succeed
        // the first two chunks.
        serverErrorList.add(ServerErrorCode.No_Error);
        serverErrorList.add(ServerErrorCode.No_Error);
        // fail requests for third and fourth data chunks including the slipped put attempts:
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        // all subsequent requests (no more puts, but there will be deletes) will succeed.
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            server.setServerErrors(serverErrorList);
        }
        // Submit the put operation and wait for it to fail.
        try {
            router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        } catch (ExecutionException e) {
            Assert.assertEquals(RouterErrorCode.AmbryUnavailable, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now, wait until the deletes of the successfully put blobs are complete.
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 44 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class RouterUtilsTest method testGetAccountContainer.

/**
 * Test {@link RouterUtils#getAccountContainer(AccountService, short, short)}.
 */
@Test
public void testGetAccountContainer() throws AccountServiceException {
    AccountService accountService = new InMemAccountService(false, false);
    // Both accountId and containerId are not tracked by AccountService.
    Pair<Account, Container> accountContainer = RouterUtils.getAccountContainer(accountService, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID);
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    accountContainer = RouterUtils.getAccountContainer(accountService, Utils.getRandomShort(random), Utils.getRandomShort(random));
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // accountId is tracked by AccountService but containerId not.
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    Account account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // Both accountId and containerId are tracked by AccountService.
    Container container = new ContainerBuilder(containerId, "ContainerNameOf" + containerId, Container.ContainerStatus.ACTIVE, "description", accountId).build();
    account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).addOrUpdateContainer(container).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container doesn't match", container, accountContainer.getSecond());
}

Example 45 with Container

use of com.github.ambry.account.Container in project ambry by linkedin.

the class RouterServerTestFramework method checkBlobId.

/**
 * Check for blob ID validity.
 * @param blobId the blobId
 * @param properties the blob properties associated with the blob (to check that the blob was put in the right
 *                   partition)
 * @param operationName a name for the operation being checked
 */
private void checkBlobId(String blobId, BlobProperties properties, String operationName) throws IOException {
    Assert.assertNotNull("Null blobId for operation: " + operationName, blobId);
    BlobId id = new BlobId(blobId, clusterMap);
    String partitionClass = MockClusterMap.DEFAULT_PARTITION_CLASS;
    Account account = accountService.getAccountById(properties.getAccountId());
    if (account != null) {
        Container container = account.getContainerById(properties.getContainerId());
        if (container != null && container.getReplicationPolicy() != null) {
            partitionClass = container.getReplicationPolicy();
        }
    }
    Assert.assertTrue("Partition that blob was put not as required by container", clusterMap.getWritablePartitionIds(partitionClass).contains(id.getPartition()));
}