Examples with XceiverClientSpi org.apache.hadoop.hdds.scm.XceiverClientSpi used on opensource projects

Example 6 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class DatanodeChunkGenerator method writeChunk.

private void writeChunk(long stepNo) throws Exception {
    // Always use this fake blockid.
    DatanodeBlockID blockId = DatanodeBlockID.newBuilder().setContainerID(1L).setLocalID(stepNo % 20).build();
    ChunkInfo chunkInfo = ChunkInfo.newBuilder().setChunkName(getPrefix() + "_testdata_chunk_" + stepNo).setOffset((stepNo / 20) * chunkSize).setLen(chunkSize).setChecksumData(checksumProtobuf).build();
    WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto.newBuilder().setBlockID(blockId).setChunkData(chunkInfo).setData(dataToWrite);
    XceiverClientSpi clientSpi = xceiverClients.get((int) (stepNo % (xceiverClients.size())));
    sendWriteChunkRequest(blockId, writeChunkRequest, clientSpi);
}

Example 7 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestOzoneClientRetriesOnExceptions method testMaxRetriesByOzoneClient.

@Test
public void testMaxRetriesByOzoneClient() throws Exception {
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, (MAX_RETRIES + 1) * blockSize);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    List<BlockOutputStreamEntry> entries = keyOutputStream.getStreamEntries();
    Assert.assertEquals((MAX_RETRIES + 1), keyOutputStream.getStreamEntries().size());
    int dataLength = maxFlushSize + 50;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    long containerID;
    List<Long> containerList = new ArrayList<>();
    for (BlockOutputStreamEntry entry : entries) {
        containerID = entry.getBlockID().getContainerID();
        ContainerInfo container = cluster.getStorageContainerManager().getContainerManager().getContainer(ContainerID.valueOf(containerID));
        Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(container.getPipelineID());
        XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline);
        Assume.assumeFalse(containerList.contains(containerID));
        containerList.add(containerID);
        xceiverClient.sendCommand(ContainerTestHelper.getCreateContainerRequest(containerID, pipeline));
        xceiverClientManager.releaseClient(xceiverClient, false);
    }
    key.write(data1);
    OutputStream stream = entries.get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    BlockOutputStream blockOutputStream = (BlockOutputStream) stream;
    TestHelper.waitForContainerClose(key, cluster);
    // Ensure that blocks for the key have been allocated to at least N+1
    // containers so that write request will be tried on N+1 different blocks
    // of N+1 different containers and it will finally fail as it will hit
    // the max retry count of N.
    Assume.assumeTrue(containerList.size() + " <= " + MAX_RETRIES, containerList.size() > MAX_RETRIES);
    try {
        key.write(data1);
        // ensure that write is flushed to dn
        key.flush();
        Assert.fail("Expected exception not thrown");
    } catch (IOException ioe) {
        Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof ContainerNotOpenException);
        Assert.assertTrue(ioe.getMessage().contains("Retry request failed. " + "retries get failed due to exceeded maximum " + "allowed retries number: " + MAX_RETRIES));
    }
    try {
        key.flush();
        Assert.fail("Expected exception not thrown");
    } catch (IOException ioe) {
        Assert.assertTrue(ioe.getMessage().contains("Stream is closed"));
    }
    try {
        key.close();
    } catch (IOException ioe) {
        Assert.fail("Expected should not be thrown");
    }
}

Example 8 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestOzoneClientRetriesOnExceptions method testGroupMismatchExceptionHandling.

@Test
public void testGroupMismatchExceptionHandling() throws Exception {
    String keyName = getKeyName();
    int dataLength = maxFlushSize + 50;
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, dataLength);
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    long containerID = keyOutputStream.getStreamEntries().get(0).getBlockID().getContainerID();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    ContainerInfo container = cluster.getStorageContainerManager().getContainerManager().getContainer(ContainerID.valueOf(containerID));
    Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(container.getPipelineID());
    XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline);
    xceiverClient.sendCommand(ContainerTestHelper.getCreateContainerRequest(containerID, pipeline));
    xceiverClientManager.releaseClient(xceiverClient, false);
    key.write(data1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    BlockOutputStream blockOutputStream = (BlockOutputStream) stream;
    TestHelper.waitForPipelineClose(key, cluster, false);
    key.flush();
    Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof GroupMismatchException);
    Assert.assertTrue(keyOutputStream.getExcludeList().getPipelineIds().contains(pipeline.getId()));
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 2);
    key.close();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 0);
    validateData(keyName, data1);
}

Example 9 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestOzoneClientRetriesOnExceptionFlushDelay method testGroupMismatchExceptionHandling.

@Test
public void testGroupMismatchExceptionHandling() throws Exception {
    String keyName = getKeyName();
    // make sure flush will sync data.
    int dataLength = maxFlushSize + chunkSize;
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, dataLength);
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    long containerID = keyOutputStream.getStreamEntries().get(0).getBlockID().getContainerID();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    ContainerInfo container = cluster.getStorageContainerManager().getContainerManager().getContainer(ContainerID.valueOf(containerID));
    Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(container.getPipelineID());
    XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline);
    xceiverClient.sendCommand(ContainerTestHelper.getCreateContainerRequest(containerID, pipeline));
    xceiverClientManager.releaseClient(xceiverClient, false);
    key.write(data1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    BlockOutputStream blockOutputStream = (BlockOutputStream) stream;
    TestHelper.waitForPipelineClose(key, cluster, false);
    key.flush();
    Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof GroupMismatchException);
    Assert.assertTrue(keyOutputStream.getExcludeList().getPipelineIds().contains(pipeline.getId()));
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 2);
    key.close();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 0);
    validateData(keyName, data1);
}

Example 10 with XceiverClientSpi

use of org.apache.hadoop.hdds.scm.XceiverClientSpi in project ozone by apache.

the class TestCommitWatcher method testReleaseBuffers.

@Test
public void testReleaseBuffers() throws Exception {
    int capacity = 2;
    BufferPool bufferPool = new BufferPool(chunkSize, capacity);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
    Pipeline pipeline = container.getPipeline();
    long containerId = container.getContainerInfo().getContainerID();
    XceiverClientSpi xceiverClient = clientManager.acquireClient(pipeline);
    Assert.assertEquals(1, xceiverClient.getRefcount());
    Assert.assertTrue(xceiverClient instanceof XceiverClientRatis);
    XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
    CommitWatcher watcher = new CommitWatcher(bufferPool, ratisClient);
    BlockID blockID = ContainerTestHelper.getTestBlockID(containerId);
    List<XceiverClientReply> replies = new ArrayList<>();
    long length = 0;
    List<CompletableFuture<ContainerProtos.ContainerCommandResponseProto>> futures = new ArrayList<>();
    for (int i = 0; i < capacity; i++) {
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, chunkSize, null);
        // add the data to the buffer pool
        final ChunkBuffer byteBuffer = bufferPool.allocateBuffer(0);
        byteBuffer.put(writeChunkRequest.getWriteChunk().getData());
        ratisClient.sendCommandAsync(writeChunkRequest);
        ContainerProtos.ContainerCommandRequestProto putBlockRequest = ContainerTestHelper.getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
        XceiverClientReply reply = ratisClient.sendCommandAsync(putBlockRequest);
        final List<ChunkBuffer> bufferList = singletonList(byteBuffer);
        length += byteBuffer.position();
        CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future = reply.getResponse().thenApply(v -> {
            watcher.updateCommitInfoMap(reply.getLogIndex(), bufferList);
            return v;
        });
        futures.add(future);
        watcher.getFutureMap().put(length, future);
        replies.add(reply);
    }
    Assert.assertTrue(replies.size() == 2);
    // wait on the 1st putBlock to complete
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future1 = futures.get(0);
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future2 = futures.get(1);
    future1.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) chunkSize).equals(future1));
    // wait on 2nd putBlock to complete
    future2.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) 2 * chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) 2 * chunkSize).equals(future2));
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 2);
    watcher.watchOnFirstIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(0).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() >= chunkSize);
    watcher.watchOnLastIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(1).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(2 * chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() == 2 * chunkSize);
    Assert.assertTrue(watcher.getFutureMap().isEmpty());
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().isEmpty());
}