Examples with RouterConfig com.github.ambry.config.RouterConfig used on opensource projects

Example 6 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class AdaptiveOperationTrackerTest method nodeLevelAdaptiveTrackerTest.

/**
 * Tests that adaptive tracker uses separate node-level histogram to determine if inflight requests are past due.
 * @throws Exception
 */
@Test
public void nodeLevelAdaptiveTrackerTest() throws Exception {
    // Mock a simple partition layout for this test: Partition1 has two replicas, one on LocalHost1 and the other on RemoteHost1;
    // Similarly, Partition2 has two replicas, one on LocalHost2 and the other on RemoteHost1.
    MockPartitionId mockPartition1 = new MockPartitionId(1L, MockClusterMap.DEFAULT_PARTITION_CLASS);
    MockPartitionId mockPartition2 = new MockPartitionId(2L, MockClusterMap.DEFAULT_PARTITION_CLASS);
    // create a new list mock datanodes instead of using the default class member
    List<Port> portList = Collections.singletonList(new Port(PORT, PortType.PLAINTEXT));
    List<String> mountPaths = Arrays.asList("mockMountPath0", "mockMountPath1", "mockMountPath2");
    MockDataNodeId localHost1 = new MockDataNodeId("LocalHost1", portList, mountPaths, "dc-0");
    MockDataNodeId localHost2 = new MockDataNodeId("LocalHost2", portList, mountPaths, "dc-0");
    MockDataNodeId remoteHost1 = new MockDataNodeId("RemoteHost1", portList, mountPaths, "dc-1");
    List<MockDataNodeId> datanodes = new ArrayList<>(Arrays.asList(localHost1, localHost2, remoteHost1));
    // distribute replicas to nodes (Note that localDC name is still "dc-0" in current setup)
    mockPartition1.replicaIds.add(new MockReplicaId(PORT, mockPartition1, localHost1, 1));
    mockPartition2.replicaIds.add(new MockReplicaId(PORT, mockPartition2, localHost2, 2));
    mockPartition1.replicaIds.add(new MockReplicaId(PORT, mockPartition1, remoteHost1, 1));
    mockPartition2.replicaIds.add(new MockReplicaId(PORT, mockPartition2, remoteHost1, 2));
    MockClusterMap clusterMap = new MockClusterMap(false, datanodes, 3, Arrays.asList(mockPartition1, mockPartition2), localDcName);
    trackerScope = OperationTrackerScope.DataNode;
    RouterConfig routerConfig = createRouterConfig(true, 1, 1, 6, null, true);
    NonBlockingRouterMetrics originalMetrics = routerMetrics;
    routerMetrics = new NonBlockingRouterMetrics(clusterMap, routerConfig);
    Counter pastDueCount = routerMetrics.getBlobPastDueCount;
    Map<Resource, CachedHistogram> localColoMap = routerMetrics.getBlobLocalDcResourceToLatency;
    Map<Resource, CachedHistogram> crossColoMap = routerMetrics.getBlobCrossDcResourceToLatency;
    // mock different latency distribution of local hosts and remote host
    Histogram localHistogram1 = localColoMap.get(localHost1);
    Histogram localHistogram2 = localColoMap.get(localHost2);
    primeTracker(localHistogram1, routerConfig.routerOperationTrackerMinDataPointsRequired, new Pair<>(0L, 50L));
    primeTracker(localHistogram2, routerConfig.routerOperationTrackerMinDataPointsRequired, new Pair<>(100L, 120L));
    double localHostCutoff1 = localHistogram1.getSnapshot().getValue(QUANTILE);
    double localHostCutoff2 = localHistogram2.getSnapshot().getValue(QUANTILE);
    OperationTracker tracker1 = getOperationTracker(routerConfig, mockPartition1);
    OperationTracker tracker2 = getOperationTracker(routerConfig, mockPartition2);
    // issue first request for both partitions in local DC
    sendRequests(tracker2, 1);
    sendRequests(tracker1, 1);
    // partition1: 0-1-0-0, partition2: 0-1-0-0
    time.sleep((long) localHostCutoff1 + 1);
    // partition1 should send 2nd request to RemoteNode1, partition2 won't because its 1st request isn't past due.
    sendRequests(tracker1, 1);
    sendRequests(tracker2, 0);
    // partition1: 0-1-0-0(local), 0-1-0-0(remote); partition2: 0-1-0-0(local), 1-0-0-0(remote)
    time.sleep((long) (localHostCutoff2 - localHostCutoff1) + 2);
    sendRequests(tracker1, 0);
    sendRequests(tracker2, 1);
    // partition1: 0-1-0-0(local), 0-1-0-0(remote); partition2: 0-1-0-0(local), 0-1-0-0(remote)
    assertFalse("Operation should not be done", tracker1.isDone() || tracker2.isDone());
    // make local requests failed
    tracker1.onResponse(partitionAndInflightReplicas.get(mockPartition1).poll(), TrackedRequestFinalState.TIMED_OUT);
    tracker2.onResponse(partitionAndInflightReplicas.get(mockPartition2).poll(), TrackedRequestFinalState.FAILURE);
    // make remote requests successful
    tracker1.onResponse(partitionAndInflightReplicas.get(mockPartition1).poll(), TrackedRequestFinalState.SUCCESS);
    tracker2.onResponse(partitionAndInflightReplicas.get(mockPartition2).poll(), TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should have succeeded", tracker1.hasSucceeded() && tracker2.hasSucceeded());
    // past due count should be 2 because requests to two local nodes didn't get response within threshold
    assertEquals("Past due counter is not expected", 2, pastDueCount.getCount());
    // number of data points in local colo histogram should be 1 because LocalHost2 finally responded FAILURE which would
    // update the histogram. Note that request to LocalHost1 became TIMED_OUT in the end which should not be counted.
    assertEquals("Mismatch in number of data points in local colo histogram", 1, routerMetrics.getBlobLocalDcLatencyMs.getCount());
    // number of data points in cross colo histogram should be 2 because both requests to RemoteHost1 succeeded and histogram
    // should be updated twice in this case.
    assertEquals("Mismatch in number of data points in cross colo histogram", 2, routerMetrics.getBlobCrossDcLatencyMs.getCount());
    // additional test: dynamically add 1 new partition and 2 new nodes. Each new node hosts a replica from new partition
    MockDataNodeId newNode1 = clusterMap.createNewDataNodes(1, "dc-0").get(0);
    MockDataNodeId newNode2 = clusterMap.createNewDataNodes(1, "dc-1").get(0);
    MockPartitionId mockPartition3 = new MockPartitionId(3L, MockClusterMap.DEFAULT_PARTITION_CLASS);
    mockPartition3.replicaIds.add(new MockReplicaId(PORT, mockPartition3, newNode1, 1));
    mockPartition3.replicaIds.add(new MockReplicaId(PORT, mockPartition3, newNode2, 2));
    OperationTracker tracker3 = getOperationTracker(routerConfig, mockPartition3);
    // send 1st request
    sendRequests(tracker3, 1);
    // attempt to send 2nd one. This will trigger router metrics to create a histogram that associated with new node
    // However, there is no 2nd request out because new created histogram doesn't of enough data points.
    sendRequests(tracker3, 0);
    // make the 1st request fail
    tracker3.onResponse(partitionAndInflightReplicas.get(mockPartition3).poll(), TrackedRequestFinalState.FAILURE);
    // 2nd request is sent
    sendRequests(tracker3, 1);
    // make the 2nd request succeed
    tracker3.onResponse(partitionAndInflightReplicas.get(mockPartition3).poll(), TrackedRequestFinalState.SUCCESS);
    assertTrue("Operation should have succeeded", tracker3.hasSucceeded());
    // restore the tracer scope and routerMetrics
    trackerScope = OperationTrackerScope.Datacenter;
    routerMetrics = originalMetrics;
}

Example 7 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class ChunkFillTest method fillChunksAndAssertSuccess.

/**
 * Create a {@link PutOperation} and pass in a channel with the blobSize set by the caller; and test the chunk
 * filling flow for puts.
 * Note that this test is for the chunk filling flow, not for the ChunkFiller thread (which never gets exercised,
 * as we do not even instantiate the {@link PutManager})
 */
private void fillChunksAndAssertSuccess() throws Exception {
    VerifiableProperties vProps = getNonBlockingRouterProperties();
    MockClusterMap mockClusterMap = new MockClusterMap();
    RouterConfig routerConfig = new RouterConfig(vProps);
    routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    ResponseHandler responseHandler = new ResponseHandler(mockClusterMap);
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    BlobProperties putBlobProperties = new BlobProperties(blobSize, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, accountId, containerId, testEncryption, null, null, null);
    Random random = new Random();
    byte[] putUserMetadata = new byte[10];
    random.nextBytes(putUserMetadata);
    putContent = new byte[blobSize];
    random.nextBytes(putContent);
    final ReadableStreamChannel putChannel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(putContent));
    FutureResult<String> futureResult = new FutureResult<String>();
    MockTime time = new MockTime();
    MockNetworkClientFactory networkClientFactory = new MockNetworkClientFactory(vProps, null, 0, 0, 0, null, time);
    if (testEncryption) {
        kms = new MockKeyManagementService(new KMSConfig(vProps), TestUtils.getRandomKey(SingleKeyManagementServiceTest.DEFAULT_KEY_SIZE_CHARS));
        cryptoService = new MockCryptoService(new CryptoServiceConfig(vProps));
        cryptoJobHandler = new CryptoJobHandler(CryptoJobHandlerTest.DEFAULT_THREAD_COUNT);
    }
    MockRouterCallback routerCallback = new MockRouterCallback(networkClientFactory.getNetworkClient(), Collections.EMPTY_LIST);
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), putUserMetadata, putChannel, PutBlobOptions.DEFAULT, futureResult, null, routerCallback, null, kms, cryptoService, cryptoJobHandler, time, putBlobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    numChunks = RouterUtils.getNumChunksForBlobAndChunkSize(blobSize, chunkSize);
    compositeBuffers = new ByteBuf[numChunks];
    compositeEncryptionKeys = new ByteBuffer[numChunks];
    compositeBlobIds = new BlobId[numChunks];
    final AtomicReference<Exception> operationException = new AtomicReference<Exception>(null);
    int chunksLeftToBeFilled = numChunks;
    do {
        if (testEncryption) {
            int chunksPerBatch = Math.min(routerConfig.routerMaxInMemPutChunks, chunksLeftToBeFilled);
            CountDownLatch onPollLatch = new CountDownLatch(chunksPerBatch);
            routerCallback.setOnPollLatch(onPollLatch);
            op.fillChunks();
            Assert.assertTrue("Latch should have been zeroed out", onPollLatch.await(1000, TimeUnit.MILLISECONDS));
            chunksLeftToBeFilled -= chunksPerBatch;
        } else {
            op.fillChunks();
        }
        // since the channel is ByteBuffer based.
        for (PutOperation.PutChunk putChunk : op.putChunks) {
            if (putChunk.isFree()) {
                continue;
            }
            Assert.assertEquals("Chunk should be ready.", PutOperation.ChunkState.Ready, putChunk.getState());
            ByteBuf buf = putChunk.buf.retainedDuplicate();
            totalSizeWritten += buf.readableBytes();
            compositeBuffers[putChunk.getChunkIndex()] = buf;
            if (testEncryption) {
                compositeEncryptionKeys[putChunk.getChunkIndex()] = putChunk.encryptedPerBlobKey.duplicate();
                compositeBlobIds[putChunk.getChunkIndex()] = putChunk.chunkBlobId;
            }
            putChunk.clear();
        }
    } while (!op.isChunkFillingDone());
    if (!testEncryption) {
        Assert.assertEquals("total size written out should match the blob size", blobSize, totalSizeWritten);
    }
    // for encrypted path, size will be implicitly tested via assertDataIdentity
    Exception exception = operationException.get();
    if (exception != null) {
        throw exception;
    }
    assertDataIdentity(mockClusterMap);
}

Example 8 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class NonBlockingRouterTest method testCompositeBlobDataChunksDeleteMaxDeleteOperation.

protected void testCompositeBlobDataChunksDeleteMaxDeleteOperation(int maxDeleteOperation) throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        if (maxDeleteOperation != 0) {
            props.setProperty(RouterConfig.ROUTER_BACKGROUND_DELETER_MAX_CONCURRENT_OPERATIONS, Integer.toString(maxDeleteOperation));
        }
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // metadata blob + data chunks.
        final AtomicReference<CountDownLatch> deletesDoneLatch = new AtomicReference<>();
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.get().countDown();
            }
        };
        NonBlockingRouterMetrics localMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
        router = new NonBlockingRouter(routerConfig, localMetrics, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        String deleteServiceId = "delete-service";
        Set<String> blobsToBeDeleted = getBlobsInServers(mockServerLayout);
        int getRequestCount = mockServerLayout.getCount(RequestOrResponseType.GetRequest);
        // The third iteration is to test the case where the blob has expired.
        for (int i = 0; i < 3; i++) {
            if (i == 2) {
                // Create a clean cluster and put another blob that immediate expires.
                setOperationParams();
                putBlobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
                blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
                Set<String> allBlobsInServer = getBlobsInServers(mockServerLayout);
                allBlobsInServer.removeAll(blobsToBeDeleted);
                blobsToBeDeleted = allBlobsInServer;
            }
            blobsThatAreDeleted.clear();
            deletesDoneLatch.set(new CountDownLatch(5));
            router.deleteBlob(blobId, deleteServiceId).get();
            Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
            Assert.assertTrue("All blobs in server are deleted", blobsThatAreDeleted.keySet().containsAll(blobsToBeDeleted));
            Assert.assertTrue("Only blobs in server are deleted", blobsToBeDeleted.containsAll(blobsThatAreDeleted.keySet()));
            for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
                String expectedServiceId = blobIdAndServiceId.getKey().equals(blobId) ? deleteServiceId : BackgroundDeleteRequest.SERVICE_ID_PREFIX + deleteServiceId;
                Assert.assertEquals("Unexpected service ID for deleted blob", expectedServiceId, blobIdAndServiceId.getValue());
            }
            // For 1 chunk deletion attempt, 1 background operation for Get is initiated which results in 2 Get Requests at
            // the servers.
            getRequestCount += 2;
            Assert.assertEquals("Only one attempt of chunk deletion should have been done", getRequestCount, mockServerLayout.getCount(RequestOrResponseType.GetRequest));
        }
        deletesDoneLatch.set(new CountDownLatch(5));
        router.deleteBlob(blobId, null).get();
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.get().await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        Assert.assertEquals("Get should NOT have been skipped", 0, localMetrics.skippedGetBlobCount.getCount());
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 9 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class NonBlockingRouterTest method testResponseHandling.

/**
 * Response handling related tests for all operation managers.
 */
@Test
public void testResponseHandling() throws Exception {
    try {
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        setOperationParams();
        final List<ReplicaId> failedReplicaIds = new ArrayList<>();
        final AtomicInteger successfulResponseCount = new AtomicInteger(0);
        final AtomicBoolean invalidResponse = new AtomicBoolean(false);
        ResponseHandler mockResponseHandler = new ResponseHandler(mockClusterMap) {

            @Override
            public void onEvent(ReplicaId replicaId, Object e) {
                if (e instanceof ServerErrorCode) {
                    if (e == ServerErrorCode.No_Error) {
                        successfulResponseCount.incrementAndGet();
                    } else {
                        invalidResponse.set(true);
                    }
                } else {
                    failedReplicaIds.add(replicaId);
                }
            }
        };
        // Instantiate a router just to put a blob successfully.
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        setRouter(props, mockServerLayout, new LoggingNotificationSystem());
        setOperationParams();
        // More extensive test for puts present elsewhere - these statements are here just to exercise the flow within the
        // NonBlockingRouter class, and to ensure that operations submitted to a router eventually completes.
        String blobIdStr = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        BlobId blobId = RouterUtils.getBlobIdFromString(blobIdStr, mockClusterMap);
        router.close();
        for (MockServer mockServer : mockServerLayout.getMockServers()) {
            mockServer.setServerErrorForAllRequests(ServerErrorCode.No_Error);
        }
        SocketNetworkClient networkClient = new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime).getNetworkClient();
        cryptoJobHandler = new CryptoJobHandler(CryptoJobHandlerTest.DEFAULT_THREAD_COUNT);
        KeyManagementService localKMS = new MockKeyManagementService(new KMSConfig(verifiableProperties), singleKeyForKMS);
        putManager = new PutManager(mockClusterMap, mockResponseHandler, new LoggingNotificationSystem(), new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap, null), new RouterCallback(networkClient, new ArrayList<>()), "0", localKMS, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        OperationHelper opHelper = new OperationHelper(OperationType.PUT);
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, -1);
        // Test that if a failed response comes before the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, 0);
        // Test that if a failed response comes after the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, null, successfulResponseCount, invalidResponse, PUT_REQUEST_PARALLELISM - 1);
        testNoResponseNoNotification(opHelper, failedReplicaIds, null, successfulResponseCount, invalidResponse);
        testResponseDeserializationError(opHelper, networkClient, null);
        opHelper = new OperationHelper(OperationType.GET);
        getManager = new GetManager(mockClusterMap, mockResponseHandler, new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap, null), new RouterCallback(networkClient, new ArrayList<BackgroundDeleteRequest>()), localKMS, cryptoService, cryptoJobHandler, mockTime);
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, -1);
        // Test that if a failed response comes before the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, 0);
        // Test that if a failed response comes after the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, GET_REQUEST_PARALLELISM - 1);
        testNoResponseNoNotification(opHelper, failedReplicaIds, blobId, successfulResponseCount, invalidResponse);
        testResponseDeserializationError(opHelper, networkClient, blobId);
        opHelper = new OperationHelper(OperationType.DELETE);
        deleteManager = new DeleteManager(mockClusterMap, mockResponseHandler, accountService, new LoggingNotificationSystem(), new RouterConfig(verifiableProperties), new NonBlockingRouterMetrics(mockClusterMap, null), new RouterCallback(null, new ArrayList<BackgroundDeleteRequest>()), mockTime);
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, -1);
        // Test that if a failed response comes before the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, 0);
        // Test that if a failed response comes after the operation is completed, failure detector is notified.
        testFailureDetectorNotification(opHelper, networkClient, failedReplicaIds, blobId, successfulResponseCount, invalidResponse, DELETE_REQUEST_PARALLELISM - 1);
        testNoResponseNoNotification(opHelper, failedReplicaIds, blobId, successfulResponseCount, invalidResponse);
        testResponseDeserializationError(opHelper, networkClient, blobId);
    } finally {
        if (putManager != null) {
            putManager.close();
        }
        if (getManager != null) {
            getManager.close();
        }
        if (deleteManager != null) {
            deleteManager.close();
        }
    }
}

Example 10 with RouterConfig

use of com.github.ambry.config.RouterConfig in project ambry by linkedin.

the class NonBlockingRouterTest method testResponseWithNullRequestInfo.

/**
 * Test the case where request is timed out in the pending queue and network client returns response with null requestInfo
 * to mark node down via response handler.
 * @throws Exception
 */
@Test
public void testResponseWithNullRequestInfo() throws Exception {
    NonBlockingRouter testRouter = null;
    try {
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
        NetworkClient mockNetworkClient = Mockito.mock(NetworkClient.class);
        Mockito.when(mockNetworkClient.warmUpConnections(anyList(), anyInt(), anyLong(), anyList())).thenReturn(1);
        doNothing().when(mockNetworkClient).close();
        List<ResponseInfo> responseInfoList = new ArrayList<>();
        MockDataNodeId testDataNode = (MockDataNodeId) mockClusterMap.getDataNodeIds().get(0);
        responseInfoList.add(new ResponseInfo(null, NetworkClientErrorCode.NetworkError, null, testDataNode));
        // By default, there are 1 operation controller and 1 background deleter thread. We set CountDownLatch to 3 so that
        // at least one thread has completed calling onResponse() and test node's state has been updated in ResponseHandler
        CountDownLatch invocationLatch = new CountDownLatch(3);
        doAnswer(invocation -> {
            invocationLatch.countDown();
            return responseInfoList;
        }).when(mockNetworkClient).sendAndPoll(anyList(), anySet(), anyInt());
        NetworkClientFactory networkClientFactory = Mockito.mock(NetworkClientFactory.class);
        Mockito.when(networkClientFactory.getNetworkClient()).thenReturn(mockNetworkClient);
        testRouter = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        assertTrue("Invocation latch didn't count to 0 within 10 seconds", invocationLatch.await(10, TimeUnit.SECONDS));
        // verify the test node is considered timeout
        assertTrue("The node should be considered timeout", testDataNode.isTimedOut());
    } finally {
        if (testRouter != null) {
            testRouter.close();
        }
    }
}