Examples with MockClusterMap com.github.ambry.clustermap.MockClusterMap used on opensource projects

Example 91 with MockClusterMap

use of com.github.ambry.clustermap.MockClusterMap in project ambry by linkedin.

the class NonBlockingRouterTest method testSuccessfulPutDataChunkDelete.

/**
 * Test that even when a composite blob put succeeds, the slipped put data chunks are deleted.
 */
@Test
public void testSuccessfulPutDataChunkDelete() throws Exception {
    try {
        // This test is somehow probabilistic. Since it is not possible to devise a mocking to enforce the occurrence of
        // slipped puts given we cannot control the order of the hosts requests are sent and not all requests are sent when
        // put requests are guaranteed to fail/succeed. So, we are setting the number of chunks and max attempts high enough
        // to guarantee that slipped puts would eventually happen and operation would succeed.
        maxPutChunkSize = PUT_CONTENT_SIZE / 8;
        final int NUM_MAX_ATTEMPTS = 100;
        Properties props = getNonBlockingRouterProperties("DC1");
        props.setProperty("router.max.slipped.put.attempts", Integer.toString(NUM_MAX_ATTEMPTS));
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // succeeds but some chunks succeed only after a retry, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(1);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        // In each DC, set up the servers such that one node always succeeds and the other nodes return an unknown_error and
        // no_error alternately. This will make it with a very high probability that there will at least be a time that a
        // put will succeed on a node but will fail on the other two.
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        for (int i = 0; i < NUM_MAX_ATTEMPTS; i++) {
            serverErrorList.add(ServerErrorCode.Unknown_Error);
            serverErrorList.add(ServerErrorCode.No_Error);
        }
        Set<String> healthyNodeDC = new HashSet<>();
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            if (healthyNodeDC.contains(dataNodeId.getDatacenterName())) {
                server.setServerErrors(serverErrorList);
            } else {
                server.resetServerErrors();
            }
            healthyNodeDC.add(dataNodeId.getDatacenterName());
        }
        // Submit the put operation and wait for it to succeed.
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        // Now, wait until at least one delete happens within AWAIT_TIMEOUT_MS.
        Assert.assertTrue("Some blobs should have been deleted within " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        // Wait for the rest of the deletes to finish.
        long waitStart = SystemTime.getInstance().milliseconds();
        while (router.getBackgroundOperationsCount() != 0 && SystemTime.getInstance().milliseconds() < waitStart + AWAIT_TIMEOUT_MS) {
            Thread.sleep(1000);
        }
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertNotSame("We should not be deleting the valid blob by mistake", blobId, blobIdAndServiceId.getKey());
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
        }
    }
}

Example 92 with MockClusterMap

use of com.github.ambry.clustermap.MockClusterMap in project ambry by linkedin.

the class NonBlockingRouterTest method testRequestResponseHandlerThreadExitFlow.

/**
 * Test RequestResponseHandler thread exit flow. If the RequestResponseHandlerThread exits on its own (due to a
 * Throwable), then the router gets closed immediately along with the completion of all the operations.
 */
@Test
public void testRequestResponseHandlerThreadExitFlow() throws Exception {
    nettyByteBufLeakHelper.setDisabled(true);
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    MockClusterMap mockClusterMap = new MockClusterMap();
    MockTime mockTime = new MockTime();
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, new MockServerLayout(mockClusterMap), mockTime), new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    assertExpectedThreadCounts(2, 1);
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowExceptionOnAllPoll);
    Future future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    try {
        while (!future.isDone()) {
            mockTime.sleep(1000);
            Thread.yield();
        }
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.OperationTimedOut, ((RouterException) e.getCause()).getErrorCode());
    }
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowThrowableOnSend);
    future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    Thread requestResponseHandlerThreadRegular = TestUtils.getThreadByThisName("RequestResponseHandlerThread-0");
    Thread requestResponseHandlerThreadBackground = TestUtils.getThreadByThisName("RequestResponseHandlerThread-backgroundDeleter");
    if (requestResponseHandlerThreadRegular != null) {
        requestResponseHandlerThreadRegular.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    if (requestResponseHandlerThreadBackground != null) {
        requestResponseHandlerThreadBackground.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    try {
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.RouterClosed, ((RouterException) e.getCause()).getErrorCode());
    }
    assertClosed();
    // Ensure that both operations failed and with the right exceptions.
    Assert.assertEquals("No ChunkFiller Thread should be running after the router is closed", 0, TestUtils.numThreadsByThisName("ChunkFillerThread"));
    Assert.assertEquals("No RequestResponseHandler should be running after the router is closed", 0, TestUtils.numThreadsByThisName("RequestResponseHandlerThread"));
    Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}

Example 93 with MockClusterMap

use of com.github.ambry.clustermap.MockClusterMap in project ambry by linkedin.

the class OperationTrackerTest method useReplicaNotSucceededSendTest.

/**
 * crossColoEnabled = true, successTarget = 1, parallelism = 2.
 * Only 4 local replicas
 *
 * 1. Get 1st local replica to send request (and sent);
 * 2. Get 2nd local replica to send request (and failed to send);
 * 3. Get 3rd local replica to send request (and sent);
 * 4. Receive 2 failed responses from the 1st and 3rd replicas;
 * 5. Get again 2nd local replica to send request (and sent);
 * 6. Get 4th local replica to send request (and failed to send);
 * 7. Receive 1 failed responses from the 2nd replicas;
 * 8. Get again 4th local replica to send request (and sent);
 * 9. Receive 1 successful response from the 4th replica;
 * 10. Operation succeeds.
 */
@Test
public void useReplicaNotSucceededSendTest() {
    int replicaCount = 4;
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = Collections.singletonList(new MockDataNodeId(portList, mountPaths, "dc-0"));
    mockPartition = new MockPartitionId();
    populateReplicaList(replicaCount, ReplicaState.STANDBY);
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    OperationTracker ot = getOperationTracker(true, 1, 2, RouterOperation.GetBlobOperation, true);
    sendRequests(ot, 2, true);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
    assertFalse("Operation should not be done", ot.isDone());
    sendRequests(ot, 1, true);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.FAILURE);
    assertFalse("Operation should not be done", ot.isDone());
    sendRequests(ot, 1, false);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should have succeeded", ot.hasSucceeded());
    assertTrue("Operation should be done", ot.isDone());
}

Example 94 with MockClusterMap

use of com.github.ambry.clustermap.MockClusterMap in project ambry by linkedin.

the class OperationTrackerTest method getOperationWithReplicaStateTest.

/**
 * Test GET operation is able to try on OFFLINE replicas if routerOperationTrackerIncludeDownReplicas is true.
 */
@Test
public void getOperationWithReplicaStateTest() {
    assumeTrue(replicasStateEnabled);
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    datanodes = new ArrayList<>(Arrays.asList(new MockDataNodeId(portList, mountPaths, "dc-0"), new MockDataNodeId(portList, mountPaths, "dc-1")));
    mockPartition = new MockPartitionId();
    for (ReplicaState state : EnumSet.of(ReplicaState.BOOTSTRAP, ReplicaState.STANDBY, ReplicaState.LEADER, ReplicaState.INACTIVE, ReplicaState.OFFLINE)) {
        populateReplicaList(1, state);
    }
    localDcName = datanodes.get(0).getDatacenterName();
    mockClusterMap = new MockClusterMap(false, datanodes, 1, Collections.singletonList(mockPartition), localDcName);
    // 1. include down replicas (OFFLINE replicas are eligible for GET)
    OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
    // make sure 4 requests fails and last one succeeds. (This is to verify operation tracker adds offline replica into replica pool as well)
    ReplicaId inflightReplica;
    for (int i = 0; i < 4; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        // verify that the first 4 replicas are not OFFLINE replica. (OFFLINE replica should be added to the end of queue)
        assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
        ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
        assertFalse("Operation should not complete", ot.isDone());
    }
    sendRequests(ot, 1, false);
    inflightReplica = inflightReplicas.poll();
    assertEquals("The last replica should be OFFLINE", ReplicaState.OFFLINE, mockPartition.replicaAndState.get(inflightReplica));
    ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should be done", ot.isDone());
    // 2. exclude down replicas
    repetitionTracker.clear();
    ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, false);
    for (int i = 0; i < 4; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        // verify that none of these replicas is OFFLINE replica.
        assertNotSame("Replica state should not be OFFLINE ", mockPartition.replicaAndState.get(inflightReplica), ReplicaState.OFFLINE);
        ot.onResponse(inflightReplica, TrackedRequestFinalState.FAILURE);
        if (i < 3) {
            assertFalse("Operation should not complete", ot.isDone());
        } else {
            assertTrue("Operation should complete", ot.isDone());
        }
    }
}

Example 95 with MockClusterMap

use of com.github.ambry.clustermap.MockClusterMap in project ambry by linkedin.

the class OperationTrackerTest method sendCrossColoRequestToDcWithMostReplicasTest.

/**
 * Test the case where originating dc name is null and operation tracker will choose dc with most replicas to send
 * cross-colo request.
 * local dc: one replica;
 * remote dc1: three replicas;
 * remote dc2: one replica;
 */
@Test
public void sendCrossColoRequestToDcWithMostReplicasTest() {
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Collections.singletonList("mockMountPath");
    // set up one node per data center for testing
    MockDataNodeId localDcNode = new MockDataNodeId(portList, mountPaths, "dc-0");
    MockDataNodeId remoteDc1Node = new MockDataNodeId(portList, mountPaths, "dc-1");
    MockDataNodeId remoteDc2Node = new MockDataNodeId(portList, mountPaths, "dc-2");
    mockPartition = new MockPartitionId();
    localDcName = localDcNode.getDatacenterName();
    originatingDcName = null;
    chooseDcWithMostReplicas = true;
    mockClusterMap = new MockClusterMap(false, Arrays.asList(localDcNode, remoteDc1Node, remoteDc2Node), 1, Collections.singletonList(mockPartition), localDcName);
    populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(localDcNode));
    populateReplicaList(3, ReplicaState.STANDBY, Collections.singletonList(remoteDc1Node));
    populateReplicaList(1, ReplicaState.STANDBY, Collections.singletonList(remoteDc2Node));
    OperationTracker ot = getOperationTracker(true, 1, 1, RouterOperation.GetBlobOperation, true);
    // make local replica return Not_Found
    sendRequests(ot, 1, false);
    ot.onResponse(inflightReplicas.poll(), TrackedRequestFinalState.NOT_FOUND);
    ReplicaId inflightReplica;
    // next, operation tracker should send request to dc-1 as it has most replicas
    for (int i = 0; i < 3; ++i) {
        sendRequests(ot, 1, false);
        inflightReplica = inflightReplicas.poll();
        assertEquals("The request should be sent to dc-1 with most replicas", "dc-1", inflightReplica.getDataNodeId().getDatacenterName());
        // we deliberately make all replicas in dc-1 return Not_Found to verify that operation won't terminate on not found
        ot.onResponse(inflightReplica, TrackedRequestFinalState.NOT_FOUND);
        assertFalse("Operation should not be done yet", ot.isDone());
    }
    // the last request should go to dc-2 and this time we make it succeed
    sendRequests(ot, 1, false);
    inflightReplica = inflightReplicas.poll();
    assertEquals("The request should be sent to dc-2", "dc-2", inflightReplica.getDataNodeId().getDatacenterName());
    ot.onResponse(inflightReplica, TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should succeed", ot.hasSucceeded());
    assertTrue("Operation should be done", ot.isDone());
}