Examples with XceiverClientRatis org.apache.hadoop.hdds.scm.XceiverClientRatis used on opensource projects

Example 16 with XceiverClientRatis

use of org.apache.hadoop.hdds.scm.XceiverClientRatis in project ozone by apache.

the class TestBlockOutputStreamWithFailuresFlushDelay method testExceptionDuringClose.

private void testExceptionDuringClose() throws Exception {
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = 167;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertTrue(blockOutputStream.getTotalAckDataLength() == 0);
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    // Since the data in the buffer is already flushed, flush here will have
    // no impact on the counters and data structures
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    // flush will make sure one more entry gets updated in the map
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);
    XceiverClientRatis raftClient = (XceiverClientRatis) blockOutputStream.getXceiverClient();
    Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
    // Close the containers on the Datanode and write more data
    TestHelper.waitForContainerClose(key, cluster);
    key.write(data1);
    // commitInfoMap will remain intact as there is no server failure
    Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
    // now close the stream, It will hit exception
    key.close();
    Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof ContainerNotOpenException);
    // Make sure the retryCount is reset after the exception is handled
    Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 0);
    // Written the same data twice
    String dataString = new String(data1, UTF_8);
    validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}

Example 17 with XceiverClientRatis

use of org.apache.hadoop.hdds.scm.XceiverClientRatis in project ozone by apache.

the class TestBlockOutputStreamWithFailuresFlushDelay method testDatanodeFailureWithPreAllocation.

@Test
public void testDatanodeFailureWithPreAllocation() throws Exception {
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 3 * blockSize, ReplicationFactor.ONE);
    int dataLength = maxFlushSize + chunkSize;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    // since its hitting the full bufferCondition, it will call watchForCommit
    // and completes at least putBlock for first flushSize worth of data
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 3);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    // we have just written data more than flush Size(2 chunks), at this time
    // buffer pool will have 3 buffers allocated worth of chunk size
    Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
    // writtenDataLength as well flushedDataLength will be updated here
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(maxFlushSize, blockOutputStream.getTotalDataFlushedLength());
    // since data equals to maxBufferSize is written, this will be a blocking
    // call and hence will wait for atleast flushSize worth of data to get
    // ack'd by all servers right here
    Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);
    // watchForCommit will clean up atleast flushSize worth of data buffer
    // where each entry corresponds to flushSize worth of data
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    // Since the data in the buffer is already flushed, flush here will have
    // no impact on the counters and data structures
    Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    // flush will make sure one more entry gets updated in the map
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);
    XceiverClientRatis raftClient = (XceiverClientRatis) blockOutputStream.getXceiverClient();
    Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
    Pipeline pipeline = raftClient.getPipeline();
    cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));
    // again write data with more than max buffer limit. This will call
    // watchForCommit again. No write will happen and
    key.write(data1);
    key.flush();
    Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof RaftRetryFailureException);
    // Make sure the retryCount is reset after the exception is handled
    Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
    Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
    // Written the same data twice
    String dataString = new String(data1, UTF_8);
    cluster.restartHddsDatanode(pipeline.getNodes().get(0), true);
    validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}

Example 18 with XceiverClientRatis

use of org.apache.hadoop.hdds.scm.XceiverClientRatis in project ozone by apache.

the class TestBlockOutputStreamWithFailuresFlushDelay method testFailureWithPrimeSizedData.

private void testFailureWithPrimeSizedData() throws Exception {
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = maxFlushSize + chunkSize;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(400, blockOutputStream.getTotalDataFlushedLength());
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    XceiverClientRatis raftClient = (XceiverClientRatis) blockOutputStream.getXceiverClient();
    Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
    // Close the containers on the Datanode and write more data
    TestHelper.waitForContainerClose(key, cluster);
    key.write(data1);
    // As a part of handling the exception, 2 failed writeChunks  will be
    // rewritten plus 1 putBlocks for flush
    // and one flush for partial chunk
    key.flush();
    Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof ContainerNotOpenException);
    // Make sure the retryCount is reset after the exception is handled
    Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
    // commitInfoMap will remain intact as there is no server failure
    Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertTrue(keyOutputStream.getLocationInfoList().size() == 0);
    // Written the same data twice
    String dataString = new String(data1, UTF_8);
    validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}

Example 19 with XceiverClientRatis

use of org.apache.hadoop.hdds.scm.XceiverClientRatis in project ozone by apache.

the class TestCommitWatcher method testReleaseBuffersOnException.

@Test
public void testReleaseBuffersOnException() throws Exception {
    int capacity = 2;
    BufferPool bufferPool = new BufferPool(chunkSize, capacity);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
    Pipeline pipeline = container.getPipeline();
    long containerId = container.getContainerInfo().getContainerID();
    XceiverClientSpi xceiverClient = clientManager.acquireClient(pipeline);
    Assert.assertEquals(1, xceiverClient.getRefcount());
    Assert.assertTrue(xceiverClient instanceof XceiverClientRatis);
    XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
    CommitWatcher watcher = new CommitWatcher(bufferPool, ratisClient);
    BlockID blockID = ContainerTestHelper.getTestBlockID(containerId);
    List<XceiverClientReply> replies = new ArrayList<>();
    long length = 0;
    List<CompletableFuture<ContainerProtos.ContainerCommandResponseProto>> futures = new ArrayList<>();
    for (int i = 0; i < capacity; i++) {
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, chunkSize, null);
        // add the data to the buffer pool
        final ChunkBuffer byteBuffer = bufferPool.allocateBuffer(0);
        byteBuffer.put(writeChunkRequest.getWriteChunk().getData());
        ratisClient.sendCommandAsync(writeChunkRequest);
        ContainerProtos.ContainerCommandRequestProto putBlockRequest = ContainerTestHelper.getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
        XceiverClientReply reply = ratisClient.sendCommandAsync(putBlockRequest);
        final List<ChunkBuffer> bufferList = singletonList(byteBuffer);
        length += byteBuffer.position();
        CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future = reply.getResponse().thenApply(v -> {
            watcher.updateCommitInfoMap(reply.getLogIndex(), bufferList);
            return v;
        });
        futures.add(future);
        watcher.getFutureMap().put(length, future);
        replies.add(reply);
    }
    Assert.assertTrue(replies.size() == 2);
    // wait on the 1st putBlock to complete
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future1 = futures.get(0);
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future2 = futures.get(1);
    future1.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) chunkSize).equals(future1));
    // wait on 2nd putBlock to complete
    future2.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) 2 * chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) 2 * chunkSize).equals(future2));
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 2);
    watcher.watchOnFirstIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(0).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() >= chunkSize);
    cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));
    cluster.shutdownHddsDatanode(pipeline.getNodes().get(1));
    try {
        // just watch for a higher index so as to ensure, it does an actual
        // call to Ratis. Otherwise, it may just return in case the commitInfoMap
        // is updated to the latest index in putBlock response.
        watcher.watchForCommit(replies.get(1).getLogIndex() + 100);
        Assert.fail("Expected exception not thrown");
    } catch (IOException ioe) {
        // with retry count set to noRetry and a lower watch request
        // timeout, watch request will eventually
        // fail with TimeoutIOException from ratis client or the client
        // can itself get AlreadyClosedException from the Ratis Server
        // and the write may fail with RaftRetryFailureException
        Throwable t = HddsClientUtils.checkForException(ioe);
        Assert.assertTrue("Unexpected exception: " + t.getClass(), t instanceof RaftRetryFailureException || t instanceof TimeoutIOException || t instanceof AlreadyClosedException || t instanceof NotReplicatedException);
    }
    if (ratisClient.getReplicatedMinCommitIndex() < replies.get(1).getLogIndex()) {
        Assert.assertTrue(watcher.getTotalAckDataLength() == chunkSize);
        Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 1);
        Assert.assertTrue(watcher.getFutureMap().size() == 1);
    } else {
        Assert.assertTrue(watcher.getTotalAckDataLength() == 2 * chunkSize);
        Assert.assertTrue(watcher.getFutureMap().isEmpty());
        Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().isEmpty());
    }
}

Example 20 with XceiverClientRatis

use of org.apache.hadoop.hdds.scm.XceiverClientRatis in project ozone by apache.

the class TestContainerStateMachineFailureOnRead method testReadStateMachineFailureClosesPipeline.

@Test(timeout = 300000)
@SuppressWarnings("squid:S3655")
public void testReadStateMachineFailureClosesPipeline() throws Exception {
    // Stop one follower datanode
    List<Pipeline> pipelines = cluster.getStorageContainerManager().getPipelineManager().getPipelines(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
    Assert.assertEquals(1, pipelines.size());
    Pipeline ratisPipeline = pipelines.iterator().next();
    Optional<HddsDatanodeService> dnToStop = cluster.getHddsDatanodes().stream().filter(s -> {
        try {
            return RatisTestHelper.isRatisFollower(s, ratisPipeline);
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
    }).findFirst();
    Assert.assertTrue(dnToStop.isPresent());
    cluster.shutdownHddsDatanode(dnToStop.get().getDatanodeDetails());
    // Verify healthy pipeline before creating key
    XceiverClientRatis xceiverClientRatis = XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf);
    xceiverClientRatis.connect();
    TestOzoneContainer.createContainerForTesting(xceiverClientRatis, 100L);
    OmKeyLocationInfo omKeyLocationInfo;
    OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.THREE, new HashMap<>());
    // First write and flush creates a container in the datanode
    key.write("ratis".getBytes(UTF_8));
    key.flush();
    // get the name of a valid container
    KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
    List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList();
    Assert.assertEquals(1, locationInfoList.size());
    omKeyLocationInfo = locationInfoList.get(0);
    key.close();
    groupOutputStream.close();
    Optional<HddsDatanodeService> leaderDn = cluster.getHddsDatanodes().stream().filter(dn -> {
        try {
            return RatisTestHelper.isRatisLeader(dn, ratisPipeline);
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
    }).findFirst();
    Assert.assertTrue(leaderDn.isPresent());
    // delete the container dir from leader
    FileUtil.fullyDelete(new File(leaderDn.get().getDatanodeStateMachine().getContainer().getContainerSet().getContainer(omKeyLocationInfo.getContainerID()).getContainerData().getContainerPath()));
    // Start the stopped datanode
    // Do not wait on restart since on stop will take long time due to
    // stale interval timeout for the test
    cluster.restartHddsDatanode(dnToStop.get().getDatanodeDetails(), false);
    cluster.waitForClusterToBeReady();
    Thread.sleep(10000);
    try {
        Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(pipelines.get(0).getId());
        Assert.assertEquals("Pipeline " + pipeline.getId() + "should be in CLOSED state", Pipeline.PipelineState.CLOSED, pipeline.getPipelineState());
    } catch (PipelineNotFoundException e) {
    // do nothing
    }
}