Search in sources :

Example 31 with Account

use of com.github.ambry.account.Account in project ambry by linkedin.

the class NonBlockingRouterTest method testCompositeBlobDataChunksDeleteMaxDeleteOperation.

protected void testCompositeBlobDataChunksDeleteMaxDeleteOperation(int maxDeleteOperation) throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        if (maxDeleteOperation != 0) {
            props.setProperty(RouterConfig.ROUTER_BACKGROUND_DELETER_MAX_CONCURRENT_OPERATIONS, Integer.toString(maxDeleteOperation));
        }
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // metadata blob + data chunks.
        final AtomicReference<CountDownLatch> deletesDoneLatch = new AtomicReference<>();
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.get().countDown();
            }
        };
        NonBlockingRouterMetrics localMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
        router = new NonBlockingRouter(routerConfig, localMetrics, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        String deleteServiceId = "delete-service";
        Set<String> blobsToBeDeleted = getBlobsInServers(mockServerLayout);
        int getRequestCount = mockServerLayout.getCount(RequestOrResponseType.GetRequest);
        // The third iteration is to test the case where the blob has expired.
        for (int i = 0; i < 3; i++) {
            if (i == 2) {
                // Create a clean cluster and put another blob that immediate expires.
                setOperationParams();
                putBlobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
                blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
                Set<String> allBlobsInServer = getBlobsInServers(mockServerLayout);
                allBlobsInServer.removeAll(blobsToBeDeleted);
                blobsToBeDeleted = allBlobsInServer;
            }
            blobsThatAreDeleted.clear();
            deletesDoneLatch.set(new CountDownLatch(5));
            router.deleteBlob(blobId, deleteServiceId).get();
            Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
            Assert.assertTrue("All blobs in server are deleted", blobsThatAreDeleted.keySet().containsAll(blobsToBeDeleted));
            Assert.assertTrue("Only blobs in server are deleted", blobsToBeDeleted.containsAll(blobsThatAreDeleted.keySet()));
            for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
                String expectedServiceId = blobIdAndServiceId.getKey().equals(blobId) ? deleteServiceId : BackgroundDeleteRequest.SERVICE_ID_PREFIX + deleteServiceId;
                Assert.assertEquals("Unexpected service ID for deleted blob", expectedServiceId, blobIdAndServiceId.getValue());
            }
            // For 1 chunk deletion attempt, 1 background operation for Get is initiated which results in 2 Get Requests at
            // the servers.
            getRequestCount += 2;
            Assert.assertEquals("Only one attempt of chunk deletion should have been done", getRequestCount, mockServerLayout.getCount(RequestOrResponseType.GetRequest));
        }
        deletesDoneLatch.set(new CountDownLatch(5));
        router.deleteBlob(blobId, null).get();
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        Assert.assertEquals("Get should NOT have been skipped", 0, localMetrics.skippedGetBlobCount.getCount());
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}
Also used : Account(com.github.ambry.account.Account) VerifiableProperties(com.github.ambry.config.VerifiableProperties) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) CountDownLatch(java.util.concurrent.CountDownLatch) RouterConfig(com.github.ambry.config.RouterConfig) Container(com.github.ambry.account.Container) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) BlobProperties(com.github.ambry.messageformat.BlobProperties) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) MockTime(com.github.ambry.utils.MockTime) MockClusterMap(com.github.ambry.clustermap.MockClusterMap)

Example 32 with Account

use of com.github.ambry.account.Account in project ambry by linkedin.

the class NonBlockingRouterTest method testSimpleBlobDelete.

/**
 * Test to ensure that for simple blob deletions, no additional background delete operations
 * are initiated.
 */
@Test
public void testSimpleBlobDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE;
        String deleteServiceId = "delete-service";
        // metadata blob + data chunks.
        final AtomicInteger deletesInitiated = new AtomicInteger();
        final AtomicReference<String> receivedDeleteServiceId = new AtomicReference<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                deletesInitiated.incrementAndGet();
                receivedDeleteServiceId.set(serviceId);
            }
        };
        Properties props = getNonBlockingRouterProperties("DC1");
        setRouter(props, new MockServerLayout(mockClusterMap), deleteTrackingNotificationSystem);
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        router.deleteBlob(blobId, deleteServiceId).get();
        long waitStart = SystemTime.getInstance().milliseconds();
        while (router.getBackgroundOperationsCount() != 0 && SystemTime.getInstance().milliseconds() < waitStart + AWAIT_TIMEOUT_MS) {
            Thread.sleep(1000);
        }
        Assert.assertEquals("All background operations should be complete ", 0, router.getBackgroundOperationsCount());
        Assert.assertEquals("Only the original blob deletion should have been initiated", 1, deletesInitiated.get());
        Assert.assertEquals("The delete service ID should match the expected value", deleteServiceId, receivedDeleteServiceId.get());
        Assert.assertEquals("Get should have been skipped", 1, routerMetrics.skippedGetBlobCount.getCount());
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}
Also used : Account(com.github.ambry.account.Account) Container(com.github.ambry.account.Container) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) AtomicReference(java.util.concurrent.atomic.AtomicReference) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) Test(org.junit.Test)

Example 33 with Account

use of com.github.ambry.account.Account in project ambry by linkedin.

the class NonBlockingRouterTest method testUnsuccessfulPutDataChunkDelete.

/**
 * Test that if a composite blob put fails, the successfully put data chunks are deleted.
 */
@Test
public void testUnsuccessfulPutDataChunkDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // fails, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(2);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        // There are 4 chunks for this blob.
        // All put operations make one request to each local server as there are 3 servers overall in the local DC.
        // Set the state of the mock servers so that they return success for the first 2 requests in order to succeed
        // the first two chunks.
        serverErrorList.add(ServerErrorCode.No_Error);
        serverErrorList.add(ServerErrorCode.No_Error);
        // fail requests for third and fourth data chunks including the slipped put attempts:
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        // all subsequent requests (no more puts, but there will be deletes) will succeed.
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            server.setServerErrors(serverErrorList);
        }
        // Submit the put operation and wait for it to fail.
        try {
            router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        } catch (ExecutionException e) {
            Assert.assertEquals(RouterErrorCode.AmbryUnavailable, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now, wait until the deletes of the successfully put blobs are complete.
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}
Also used : Account(com.github.ambry.account.Account) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) Container(com.github.ambry.account.Container) ExecutionException(java.util.concurrent.ExecutionException) MockTime(com.github.ambry.utils.MockTime) VerifiableProperties(com.github.ambry.config.VerifiableProperties) CountDownLatch(java.util.concurrent.CountDownLatch) RouterConfig(com.github.ambry.config.RouterConfig) ServerErrorCode(com.github.ambry.server.ServerErrorCode) LoggingNotificationSystem(com.github.ambry.commons.LoggingNotificationSystem) DataNodeId(com.github.ambry.clustermap.DataNodeId) MockDataNodeId(com.github.ambry.clustermap.MockDataNodeId) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 34 with Account

use of com.github.ambry.account.Account in project ambry by linkedin.

the class RouterUtilsTest method testGetAccountContainer.

/**
 * Test {@link RouterUtils#getAccountContainer(AccountService, short, short)}.
 */
@Test
public void testGetAccountContainer() throws AccountServiceException {
    AccountService accountService = new InMemAccountService(false, false);
    // Both accountId and containerId are not tracked by AccountService.
    Pair<Account, Container> accountContainer = RouterUtils.getAccountContainer(accountService, Account.UNKNOWN_ACCOUNT_ID, Container.UNKNOWN_CONTAINER_ID);
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    accountContainer = RouterUtils.getAccountContainer(accountService, Utils.getRandomShort(random), Utils.getRandomShort(random));
    Assert.assertEquals("Account should be null", null, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // accountId is tracked by AccountService but containerId not.
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    Account account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container should be null", null, accountContainer.getSecond());
    // Both accountId and containerId are tracked by AccountService.
    Container container = new ContainerBuilder(containerId, "ContainerNameOf" + containerId, Container.ContainerStatus.ACTIVE, "description", accountId).build();
    account = new AccountBuilder(accountId, "AccountNameOf" + accountId, Account.AccountStatus.ACTIVE).addOrUpdateContainer(container).build();
    accountService.updateAccounts(Arrays.asList(account));
    accountContainer = RouterUtils.getAccountContainer(accountService, accountId, containerId);
    Assert.assertEquals("Account doesn't match", account, accountContainer.getFirst());
    Assert.assertEquals("Container doesn't match", container, accountContainer.getSecond());
}
Also used : Account(com.github.ambry.account.Account) InMemAccountService(com.github.ambry.account.InMemAccountService) Container(com.github.ambry.account.Container) ContainerBuilder(com.github.ambry.account.ContainerBuilder) AccountBuilder(com.github.ambry.account.AccountBuilder) AccountService(com.github.ambry.account.AccountService) InMemAccountService(com.github.ambry.account.InMemAccountService) Test(org.junit.Test)

Example 35 with Account

use of com.github.ambry.account.Account in project ambry by linkedin.

the class RouterServerTestFramework method checkBlobId.

/**
 * Check for blob ID validity.
 * @param blobId the blobId
 * @param properties the blob properties associated with the blob (to check that the blob was put in the right
 *                   partition)
 * @param operationName a name for the operation being checked
 */
private void checkBlobId(String blobId, BlobProperties properties, String operationName) throws IOException {
    Assert.assertNotNull("Null blobId for operation: " + operationName, blobId);
    BlobId id = new BlobId(blobId, clusterMap);
    String partitionClass = MockClusterMap.DEFAULT_PARTITION_CLASS;
    Account account = accountService.getAccountById(properties.getAccountId());
    if (account != null) {
        Container container = account.getContainerById(properties.getContainerId());
        if (container != null && container.getReplicationPolicy() != null) {
            partitionClass = container.getReplicationPolicy();
        }
    }
    Assert.assertTrue("Partition that blob was put not as required by container", clusterMap.getWritablePartitionIds(partitionClass).contains(id.getPartition()));
}
Also used : Account(com.github.ambry.account.Account) Container(com.github.ambry.account.Container) BlobId(com.github.ambry.commons.BlobId)

Aggregations

Account (com.github.ambry.account.Account)114 Container (com.github.ambry.account.Container)87 Test (org.junit.Test)67 RestServiceException (com.github.ambry.rest.RestServiceException)24 ArrayList (java.util.ArrayList)22 RestRequest (com.github.ambry.rest.RestRequest)18 JSONObject (org.json.JSONObject)18 MockRestRequest (com.github.ambry.rest.MockRestRequest)17 VerifiableProperties (com.github.ambry.config.VerifiableProperties)16 HashMap (java.util.HashMap)15 HashSet (java.util.HashSet)15 AccountBuilder (com.github.ambry.account.AccountBuilder)14 MockRestResponseChannel (com.github.ambry.rest.MockRestResponseChannel)14 ContainerBuilder (com.github.ambry.account.ContainerBuilder)13 Properties (java.util.Properties)13 MetricRegistry (com.codahale.metrics.MetricRegistry)12 InMemAccountService (com.github.ambry.account.InMemAccountService)12 ByteBuffer (java.nio.ByteBuffer)12 RestMethod (com.github.ambry.rest.RestMethod)11 Map (java.util.Map)11