Examples with XceiverClientSpi org.apache.hadoop.hdds.scm.XceiverClientSpi used on opensource projects

Example 26 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class DatanodeChunkGenerator method call.

@Override
public Void call() throws Exception {
    OzoneConfiguration ozoneConf = createOzoneConfiguration();
    if (OzoneSecurityUtil.isSecurityEnabled(ozoneConf)) {
        throw new IllegalArgumentException("Datanode chunk generator is not supported in secure environment");
    }
    List<String> pipelinesFromCmd = Arrays.asList(pipelineIds.split(","));
    List<String> datanodeHosts = Arrays.asList(this.datanodes.split(","));
    Set<Pipeline> pipelines;
    try (StorageContainerLocationProtocol scmLocationClient = createStorageContainerLocationClient(ozoneConf);
        XceiverClientManager xceiverClientManager = new XceiverClientManager(ozoneConf)) {
        List<Pipeline> pipelinesFromSCM = scmLocationClient.listPipelines();
        Pipeline firstPipeline;
        init();
        if (!arePipelinesOrDatanodesProvided()) {
            // default behaviour if no arguments provided
            firstPipeline = pipelinesFromSCM.stream().filter(p -> p.getReplicationConfig().getRequiredNodes() == 3).findFirst().orElseThrow(() -> new IllegalArgumentException("Pipeline ID is NOT defined, and no pipeline " + "has been found with factor=THREE"));
            XceiverClientSpi xceiverClientSpi = xceiverClientManager.acquireClient(firstPipeline);
            xceiverClients = new ArrayList<>();
            xceiverClients.add(xceiverClientSpi);
        } else {
            xceiverClients = new ArrayList<>();
            pipelines = new HashSet<>();
            for (String pipelineId : pipelinesFromCmd) {
                List<Pipeline> selectedPipelines = pipelinesFromSCM.stream().filter((p -> p.getId().toString().equals("PipelineID=" + pipelineId) || pipelineContainsDatanode(p, datanodeHosts))).collect(Collectors.toList());
                pipelines.addAll(selectedPipelines);
            }
            for (Pipeline p : pipelines) {
                LOG.info("Writing to pipeline: " + p.getId());
                xceiverClients.add(xceiverClientManager.acquireClient(p));
            }
            if (pipelines.isEmpty()) {
                throw new IllegalArgumentException("Couldn't find the any/the selected pipeline");
            }
        }
        runTest();
    } finally {
        for (XceiverClientSpi xceiverClientSpi : xceiverClients) {
            if (xceiverClientSpi != null) {
                xceiverClientSpi.close();
            }
        }
    }
    return null;
}

Example 27 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestCommitWatcher method testReleaseBuffersOnException.

@Test
public void testReleaseBuffersOnException() throws Exception {
    int capacity = 2;
    BufferPool bufferPool = new BufferPool(chunkSize, capacity);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
    Pipeline pipeline = container.getPipeline();
    long containerId = container.getContainerInfo().getContainerID();
    XceiverClientSpi xceiverClient = clientManager.acquireClient(pipeline);
    Assert.assertEquals(1, xceiverClient.getRefcount());
    Assert.assertTrue(xceiverClient instanceof XceiverClientRatis);
    XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
    CommitWatcher watcher = new CommitWatcher(bufferPool, ratisClient);
    BlockID blockID = ContainerTestHelper.getTestBlockID(containerId);
    List<XceiverClientReply> replies = new ArrayList<>();
    long length = 0;
    List<CompletableFuture<ContainerProtos.ContainerCommandResponseProto>> futures = new ArrayList<>();
    for (int i = 0; i < capacity; i++) {
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, chunkSize, null);
        // add the data to the buffer pool
        final ChunkBuffer byteBuffer = bufferPool.allocateBuffer(0);
        byteBuffer.put(writeChunkRequest.getWriteChunk().getData());
        ratisClient.sendCommandAsync(writeChunkRequest);
        ContainerProtos.ContainerCommandRequestProto putBlockRequest = ContainerTestHelper.getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
        XceiverClientReply reply = ratisClient.sendCommandAsync(putBlockRequest);
        final List<ChunkBuffer> bufferList = singletonList(byteBuffer);
        length += byteBuffer.position();
        CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future = reply.getResponse().thenApply(v -> {
            watcher.updateCommitInfoMap(reply.getLogIndex(), bufferList);
            return v;
        });
        futures.add(future);
        watcher.getFutureMap().put(length, future);
        replies.add(reply);
    }
    Assert.assertTrue(replies.size() == 2);
    // wait on the 1st putBlock to complete
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future1 = futures.get(0);
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future2 = futures.get(1);
    future1.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) chunkSize).equals(future1));
    // wait on 2nd putBlock to complete
    future2.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) 2 * chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) 2 * chunkSize).equals(future2));
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 2);
    watcher.watchOnFirstIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(0).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() >= chunkSize);
    cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));
    cluster.shutdownHddsDatanode(pipeline.getNodes().get(1));
    try {
        // just watch for a higher index so as to ensure, it does an actual
        // call to Ratis. Otherwise, it may just return in case the commitInfoMap
        // is updated to the latest index in putBlock response.
        watcher.watchForCommit(replies.get(1).getLogIndex() + 100);
        Assert.fail("Expected exception not thrown");
    } catch (IOException ioe) {
        // with retry count set to noRetry and a lower watch request
        // timeout, watch request will eventually
        // fail with TimeoutIOException from ratis client or the client
        // can itself get AlreadyClosedException from the Ratis Server
        // and the write may fail with RaftRetryFailureException
        Throwable t = HddsClientUtils.checkForException(ioe);
        Assert.assertTrue("Unexpected exception: " + t.getClass(), t instanceof RaftRetryFailureException || t instanceof TimeoutIOException || t instanceof AlreadyClosedException || t instanceof NotReplicatedException);
    }
    if (ratisClient.getReplicatedMinCommitIndex() < replies.get(1).getLogIndex()) {
        Assert.assertTrue(watcher.getTotalAckDataLength() == chunkSize);
        Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 1);
        Assert.assertTrue(watcher.getFutureMap().size() == 1);
    } else {
        Assert.assertTrue(watcher.getTotalAckDataLength() == 2 * chunkSize);
        Assert.assertTrue(watcher.getFutureMap().isEmpty());
        Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().isEmpty());
    }
}

Example 28 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestContainerStateMachineFailures method testWriteStateMachineDataIdempotencyWithClosedContainer.

// The test injects multiple write chunk requests along with closed container
// request thereby inducing a situation where a writeStateMachine call
// gets executed when the closed container apply completes thereby
// failing writeStateMachine call. In any case, our stateMachine should
// not be marked unhealthy and pipeline should not fail if container gets
// closed here.
@Test
public void testWriteStateMachineDataIdempotencyWithClosedContainer() throws Exception {
    OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey("ratis-1", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    // First write and flush creates a container in the datanode
    key.write("ratis".getBytes(UTF_8));
    key.flush();
    key.write("ratis".getBytes(UTF_8));
    KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
    List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList();
    Assert.assertEquals(1, locationInfoList.size());
    OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
    HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, cluster);
    ContainerData containerData = dn.getDatanodeStateMachine().getContainer().getContainerSet().getContainer(omKeyLocationInfo.getContainerID()).getContainerData();
    Assert.assertTrue(containerData instanceof KeyValueContainerData);
    key.close();
    ContainerStateMachine stateMachine = (ContainerStateMachine) TestHelper.getStateMachine(dn, omKeyLocationInfo.getPipeline());
    SimpleStateMachineStorage storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
    Path parentPath = storage.findLatestSnapshot().getFile().getPath();
    stateMachine.takeSnapshot();
    // Since the snapshot threshold is set to 1, since there are
    // applyTransactions, we should see snapshots
    Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
    FileInfo snapshot = storage.findLatestSnapshot().getFile();
    Assert.assertNotNull(snapshot);
    long containerID = omKeyLocationInfo.getContainerID();
    Pipeline pipeline = cluster.getStorageContainerLocationClient().getContainerWithPipeline(containerID).getPipeline();
    XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline);
    CountDownLatch latch = new CountDownLatch(100);
    int count = 0;
    AtomicInteger failCount = new AtomicInteger(0);
    Runnable r1 = () -> {
        try {
            ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder();
            request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
            request.setCmdType(ContainerProtos.Type.CloseContainer);
            request.setContainerID(containerID);
            request.setCloseContainer(ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
            xceiverClient.sendCommand(request.build());
        } catch (IOException e) {
            failCount.incrementAndGet();
        }
    };
    Runnable r2 = () -> {
        try {
            ByteString data = ByteString.copyFromUtf8("hello");
            ContainerProtos.ContainerCommandRequestProto.Builder writeChunkRequest = ContainerTestHelper.newWriteChunkRequestBuilder(pipeline, omKeyLocationInfo.getBlockID(), data.size(), random.nextInt());
            writeChunkRequest.setWriteChunk(writeChunkRequest.getWriteChunkBuilder().setData(data));
            xceiverClient.sendCommand(writeChunkRequest.build());
            latch.countDown();
        } catch (IOException e) {
            latch.countDown();
            if (!(HddsClientUtils.checkForException(e) instanceof ContainerNotOpenException)) {
                failCount.incrementAndGet();
            }
            String message = e.getMessage();
            Assert.assertFalse(message, message.contains("hello"));
            Assert.assertTrue(message, message.contains(HddsUtils.REDACTED.toStringUtf8()));
        }
    };
    List<Thread> threadList = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        count++;
        Thread r = new Thread(r2);
        r.start();
        threadList.add(r);
    }
    Thread closeContainerThread = new Thread(r1);
    closeContainerThread.start();
    threadList.add(closeContainerThread);
    latch.await(600, TimeUnit.SECONDS);
    for (int i = 0; i < 101; i++) {
        threadList.get(i).join();
    }
    if (failCount.get() > 0) {
        fail("testWriteStateMachineDataIdempotencyWithClosedContainer failed");
    }
    Assert.assertTrue(TestHelper.getDatanodeService(omKeyLocationInfo, cluster).getDatanodeStateMachine().getContainer().getContainerSet().getContainer(containerID).getContainerState() == ContainerProtos.ContainerDataProto.State.CLOSED);
    Assert.assertTrue(stateMachine.isStateMachineHealthy());
    try {
        stateMachine.takeSnapshot();
    } catch (IOException ioe) {
        Assert.fail("Exception should not be thrown");
    }
    FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
    Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
    r2.run();
}

Example 29 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestDeleteWithSlowFollower method testDeleteKeyWithSlowFollower.

/**
 * The test simulates a slow follower by first writing key thereby creating a
 * a container on 3 dns of the cluster. Then, a dn is shutdown and a close
 * container cmd gets issued so that in the leader and the alive follower,
 * container gets closed. And then, key is deleted and
 * the node is started up again so that it
 * rejoins the ring and starts applying the transaction from where it left
 * by fetching the entries from the leader. Until and unless this follower
 * catches up and its replica gets closed,
 * the data is not deleted from any of the nodes which have the
 * closed replica.
 */
@Test
public void testDeleteKeyWithSlowFollower() throws Exception {
    String keyName = "ratis";
    OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey(keyName, 0, ReplicationType.RATIS, ReplicationFactor.THREE, new HashMap<>());
    byte[] testData = "ratis".getBytes(UTF_8);
    // First write and flush creates a container in the datanode
    key.write(testData);
    key.flush();
    KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
    List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList();
    Assume.assumeTrue("Expected exactly a single location, but got: " + locationInfoList.size(), 1 == locationInfoList.size());
    OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
    long containerID = omKeyLocationInfo.getContainerID();
    // A container is created on the datanode. Now figure out a follower node to
    // kill/slow down.
    HddsDatanodeService follower = null;
    HddsDatanodeService leader = null;
    List<Pipeline> pipelineList = cluster.getStorageContainerManager().getPipelineManager().getPipelines(RatisReplicationConfig.getInstance(THREE));
    Assume.assumeTrue(pipelineList.size() >= FACTOR_THREE_PIPELINE_COUNT);
    Pipeline pipeline = pipelineList.get(0);
    for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
        if (RatisTestHelper.isRatisFollower(dn, pipeline)) {
            follower = dn;
        } else if (RatisTestHelper.isRatisLeader(dn, pipeline)) {
            leader = dn;
        }
    }
    Assume.assumeNotNull(follower, leader);
    // ensure that the chosen follower is still a follower
    Assume.assumeTrue(RatisTestHelper.isRatisFollower(follower, pipeline));
    // shutdown the  follower node
    cluster.shutdownHddsDatanode(follower.getDatanodeDetails());
    key.write(testData);
    key.close();
    // now move the container to the closed on the datanode.
    XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline);
    ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder();
    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
    request.setCmdType(ContainerProtos.Type.CloseContainer);
    request.setContainerID(containerID);
    request.setCloseContainer(ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
    xceiverClient.sendCommand(request.build());
    ContainerStateMachine stateMachine = (ContainerStateMachine) RatisTestHelper.getStateMachine(leader, pipeline);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setReplicationConfig(RatisReplicationConfig.getInstance(THREE)).setKeyName(keyName).build();
    OmKeyInfo info = cluster.getOzoneManager().lookupKey(keyArgs);
    BlockID blockID = info.getKeyLocationVersions().get(0).getLocationList().get(0).getBlockID();
    OzoneContainer ozoneContainer;
    final DatanodeStateMachine dnStateMachine = leader.getDatanodeStateMachine();
    ozoneContainer = dnStateMachine.getContainer();
    KeyValueHandler keyValueHandler = (KeyValueHandler) ozoneContainer.getDispatcher().getHandler(ContainerProtos.ContainerType.KeyValueContainer);
    Container container = ozoneContainer.getContainerSet().getContainer(blockID.getContainerID());
    KeyValueContainerData containerData = ((KeyValueContainerData) container.getContainerData());
    long delTrxId = containerData.getDeleteTransactionId();
    long numPendingDeletionBlocks = containerData.getNumPendingDeletionBlocks();
    BlockData blockData = keyValueHandler.getBlockManager().getBlock(container, blockID);
    // cluster.getOzoneManager().deleteKey(keyArgs);
    client.getObjectStore().getVolume(volumeName).getBucket(bucketName).deleteKey("ratis");
    GenericTestUtils.waitFor(() -> {
        try {
            if (SCMHAUtils.isSCMHAEnabled(cluster.getConf())) {
                cluster.getStorageContainerManager().getScmHAManager().asSCMHADBTransactionBuffer().flush();
            }
            return dnStateMachine.getCommandDispatcher().getDeleteBlocksCommandHandler().getInvocationCount() >= 1;
        } catch (IOException e) {
            return false;
        }
    }, 500, 100000);
    Assert.assertTrue(containerData.getDeleteTransactionId() > delTrxId);
    Assert.assertTrue(containerData.getNumPendingDeletionBlocks() > numPendingDeletionBlocks);
    // deleteBlock handler is invoked
    try {
        for (ContainerProtos.ChunkInfo chunkInfo : blockData.getChunks()) {
            keyValueHandler.getChunkManager().readChunk(container, blockID, ChunkInfo.getFromProtoBuf(chunkInfo), null);
        }
    } catch (IOException ioe) {
        Assert.fail("Exception should not be thrown.");
    }
    long numReadStateMachineOps = stateMachine.getMetrics().getNumReadStateMachineOps();
    Assert.assertTrue(stateMachine.getMetrics().getNumReadStateMachineFails() == 0);
    stateMachine.evictStateMachineCache();
    cluster.restartHddsDatanode(follower.getDatanodeDetails(), false);
    // wait for the raft server to come up and join the ratis ring
    Thread.sleep(10000);
    // Make sure the readStateMachine call got triggered after the follower
    // caught up
    Assert.assertTrue(stateMachine.getMetrics().getNumReadStateMachineOps() > numReadStateMachineOps);
    Assert.assertTrue(stateMachine.getMetrics().getNumReadStateMachineFails() == 0);
    // wait for the chunk to get deleted now
    Thread.sleep(10000);
    for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
        keyValueHandler = (KeyValueHandler) dn.getDatanodeStateMachine().getContainer().getDispatcher().getHandler(ContainerProtos.ContainerType.KeyValueContainer);
        // make sure the chunk is now deleted on the all dns
        try {
            for (ContainerProtos.ChunkInfo chunkInfo : blockData.getChunks()) {
                keyValueHandler.getChunkManager().readChunk(container, blockID, ChunkInfo.getFromProtoBuf(chunkInfo), null);
            }
            Assert.fail("Expected exception is not thrown");
        } catch (IOException ioe) {
            Assert.assertTrue(ioe instanceof StorageContainerException);
            Assert.assertTrue(((StorageContainerException) ioe).getResult() == ContainerProtos.Result.UNABLE_TO_FIND_CHUNK);
        }
    }
}

Example 30 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestCSMMetrics method runContainerStateMachineMetrics.

static void runContainerStateMachineMetrics(int numDatanodes, BiConsumer<Pipeline, OzoneConfiguration> initConf, TestCSMMetrics.CheckedBiFunction<Pipeline, OzoneConfiguration, XceiverClientSpi, IOException> createClient, TestCSMMetrics.CheckedBiFunction<DatanodeDetails, OzoneConfiguration, XceiverServerSpi, IOException> createServer, CheckedBiConsumer<DatanodeDetails, Pipeline, IOException> initServer) throws Exception {
    final List<XceiverServerSpi> servers = new ArrayList<>();
    XceiverClientSpi client = null;
    try {
        final Pipeline pipeline = MockPipeline.createPipeline(numDatanodes);
        final OzoneConfiguration conf = new OzoneConfiguration();
        initConf.accept(pipeline, conf);
        for (DatanodeDetails dn : pipeline.getNodes()) {
            final XceiverServerSpi s = createServer.apply(dn, conf);
            servers.add(s);
            s.start();
            initServer.accept(dn, pipeline);
        }
        client = createClient.apply(pipeline, conf);
        client.connect();
        // Before Read Chunk/Write Chunk
        MetricsRecordBuilder metric = getMetrics(CSMMetrics.SOURCE_NAME + RaftGroupId.valueOf(pipeline.getId().getId()).toString());
        assertCounter("NumWriteStateMachineOps", 0L, metric);
        assertCounter("NumReadStateMachineOps", 0L, metric);
        assertCounter("NumApplyTransactionOps", 0L, metric);
        assertCounter("NumBytesWrittenCount", 0L, metric);
        assertCounter("NumBytesCommittedCount", 0L, metric);
        assertCounter("NumStartTransactionVerifyFailures", 0L, metric);
        assertCounter("NumContainerNotOpenVerifyFailures", 0L, metric);
        assertCounter("WriteChunkNumOps", 0L, metric);
        double applyTransactionLatency = getDoubleGauge("ApplyTransactionAvgTime", metric);
        assertTrue(applyTransactionLatency == 0.0);
        double writeStateMachineLatency = getDoubleGauge("WriteStateMachineDataAvgTime", metric);
        assertTrue(writeStateMachineLatency == 0.0);
        // Write Chunk
        BlockID blockID = ContainerTestHelper.getTestBlockID(ContainerTestHelper.getTestContainerID());
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, 1024, null);
        ContainerCommandResponseProto response = client.sendCommand(writeChunkRequest);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
        metric = getMetrics(CSMMetrics.SOURCE_NAME + RaftGroupId.valueOf(pipeline.getId().getId()).toString());
        assertCounter("NumWriteStateMachineOps", 1L, metric);
        assertCounter("NumBytesWrittenCount", 1024L, metric);
        assertCounter("NumApplyTransactionOps", 1L, metric);
        assertCounter("NumBytesCommittedCount", 1024L, metric);
        assertCounter("NumStartTransactionVerifyFailures", 0L, metric);
        assertCounter("NumContainerNotOpenVerifyFailures", 0L, metric);
        assertCounter("WriteChunkNumOps", 1L, metric);
        // Read Chunk
        ContainerProtos.ContainerCommandRequestProto readChunkRequest = ContainerTestHelper.getReadChunkRequest(pipeline, writeChunkRequest.getWriteChunk());
        response = client.sendCommand(readChunkRequest);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
        metric = getMetrics(CSMMetrics.SOURCE_NAME + RaftGroupId.valueOf(pipeline.getId().getId()).toString());
        assertCounter("NumQueryStateMachineOps", 1L, metric);
        assertCounter("NumApplyTransactionOps", 1L, metric);
        applyTransactionLatency = getDoubleGauge("ApplyTransactionAvgTime", metric);
        assertTrue(applyTransactionLatency > 0.0);
        writeStateMachineLatency = getDoubleGauge("WriteStateMachineDataAvgTime", metric);
        assertTrue(writeStateMachineLatency > 0.0);
    } finally {
        if (client != null) {
            client.close();
        }
        servers.stream().forEach(XceiverServerSpi::stop);
    }
}