Examples with ContainerCommandRequestProto org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto used on opensource projects

Example 1 with ContainerCommandRequestProto

use of org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto in project ozone by apache.

the class TestXceiverClientMetrics method testMetrics.

@Test
public void testMetrics() throws Exception {
    OzoneConfiguration conf = new OzoneConfiguration();
    String metaDir = GenericTestUtils.getTempPath(TestXceiverClientManager.class.getName() + UUID.randomUUID());
    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(SCMTestUtils.getReplicationType(conf), SCMTestUtils.getReplicationFactor(conf), OzoneConsts.OZONE);
    XceiverClientSpi client = clientManager.acquireClient(container.getPipeline());
    ContainerCommandRequestProto request = ContainerTestHelper.getCreateContainerRequest(container.getContainerInfo().getContainerID(), container.getPipeline());
    client.sendCommand(request);
    MetricsRecordBuilder containerMetrics = getMetrics(XceiverClientMetrics.SOURCE_NAME);
    // Above request command is in a synchronous way, so there will be no
    // pending requests.
    assertCounter("PendingOps", 0L, containerMetrics);
    assertCounter("numPendingCreateContainer", 0L, containerMetrics);
    // the counter value of average latency metric should be increased
    assertCounter("CreateContainerLatencyNumOps", 1L, containerMetrics);
    breakFlag = false;
    latch = new CountDownLatch(1);
    int numRequest = 10;
    List<CompletableFuture<ContainerCommandResponseProto>> computeResults = new ArrayList<>();
    // start new thread to send async requests
    Thread sendThread = new Thread(() -> {
        while (!breakFlag) {
            try {
                // use async interface for testing pending metrics
                for (int i = 0; i < numRequest; i++) {
                    BlockID blockID = ContainerTestHelper.getTestBlockID(container.getContainerInfo().getContainerID());
                    ContainerProtos.ContainerCommandRequestProto smallFileRequest;
                    smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest(client.getPipeline(), blockID, 1024);
                    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> response = client.sendCommandAsync(smallFileRequest).getResponse();
                    computeResults.add(response);
                }
                Thread.sleep(1000);
            } catch (Exception ignored) {
            }
        }
        latch.countDown();
    });
    sendThread.start();
    GenericTestUtils.waitFor(() -> {
        // check if pending metric count is increased
        MetricsRecordBuilder metric = getMetrics(XceiverClientMetrics.SOURCE_NAME);
        long pendingOps = getLongCounter("PendingOps", metric);
        long pendingPutSmallFileOps = getLongCounter("numPendingPutSmallFile", metric);
        if (pendingOps > 0 && pendingPutSmallFileOps > 0) {
            // reset break flag
            breakFlag = true;
            return true;
        } else {
            return false;
        }
    }, 100, 60000);
    // blocking until we stop sending async requests
    latch.await();
    // Wait for all futures being done.
    GenericTestUtils.waitFor(() -> {
        for (CompletableFuture future : computeResults) {
            if (!future.isDone()) {
                return false;
            }
        }
        return true;
    }, 100, 60000);
    // the counter value of pending metrics should be decreased to 0
    containerMetrics = getMetrics(XceiverClientMetrics.SOURCE_NAME);
    assertCounter("PendingOps", 0L, containerMetrics);
    assertCounter("numPendingPutSmallFile", 0L, containerMetrics);
    clientManager.close();
}

Example 2 with ContainerCommandRequestProto

use of org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto in project ozone by apache.

the class DatanodeChunkValidator method readReference.

/**
 * Read a reference chunk using same name than one from the
 * {@link org.apache.hadoop.ozone.freon.DatanodeChunkGenerator}.
 */
private void readReference() throws IOException {
    ContainerCommandRequestProto request = createReadChunkRequest(0);
    ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
    checksum = new Checksum(ContainerProtos.ChecksumType.CRC32, chunkSize);
    checksumReference = computeChecksum(response);
}

Example 3 with ContainerCommandRequestProto

use of org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto in project ozone by apache.

the class DatanodeChunkValidator method validateChunk.

private void validateChunk(long stepNo) throws Exception {
    ContainerCommandRequestProto request = createReadChunkRequest(stepNo);
    timer.time(() -> {
        try {
            ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
            ChecksumData checksumOfChunk = computeChecksum(response);
            if (!checksumReference.equals(checksumOfChunk)) {
                throw new IllegalStateException("Reference (=first) message checksum doesn't match " + "with checksum of chunk " + response.getReadChunk().getChunkData().getChunkName());
            }
        } catch (IOException e) {
            LOG.warn("Could not read chunk due to IOException: ", e);
        }
    });
}

Example 4 with ContainerCommandRequestProto

use of org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto in project ozone by apache.

the class CloseContainerCommandHandler method handle.

/**
 * Handles a given SCM command.
 *
 * @param command           - SCM Command
 * @param ozoneContainer    - Ozone Container.
 * @param context           - Current Context.
 * @param connectionManager - The SCMs that we are talking to.
 */
@Override
public void handle(SCMCommand command, OzoneContainer ozoneContainer, StateContext context, SCMConnectionManager connectionManager) {
    invocationCount.incrementAndGet();
    final long startTime = Time.monotonicNow();
    final DatanodeDetails datanodeDetails = context.getParent().getDatanodeDetails();
    final CloseContainerCommandProto closeCommand = ((CloseContainerCommand) command).getProto();
    final ContainerController controller = ozoneContainer.getController();
    final long containerId = closeCommand.getContainerID();
    LOG.debug("Processing Close Container command container #{}", containerId);
    try {
        final Container container = controller.getContainer(containerId);
        if (container == null) {
            LOG.error("Container #{} does not exist in datanode. " + "Container close failed.", containerId);
            return;
        }
        // move the container to CLOSING if in OPEN state
        controller.markContainerForClose(containerId);
        switch(container.getContainerState()) {
            case OPEN:
            case CLOSING:
                // If the container is part of open pipeline, close it via write channel
                if (ozoneContainer.getWriteChannel().isExist(closeCommand.getPipelineID())) {
                    ContainerCommandRequestProto request = getContainerCommandRequestProto(datanodeDetails, closeCommand.getContainerID(), command.getEncodedToken());
                    ozoneContainer.getWriteChannel().submitRequest(request, closeCommand.getPipelineID());
                } else {
                    controller.quasiCloseContainer(containerId);
                    LOG.info("Marking Container {} quasi closed", containerId);
                }
                break;
            case QUASI_CLOSED:
                if (closeCommand.getForce()) {
                    controller.closeContainer(containerId);
                }
                break;
            case CLOSED:
                break;
            case UNHEALTHY:
            case INVALID:
                LOG.debug("Cannot close the container #{}, the container is" + " in {} state.", containerId, container.getContainerState());
                break;
            default:
                break;
        }
    } catch (NotLeaderException e) {
        LOG.debug("Follower cannot close container #{}.", containerId);
    } catch (IOException e) {
        LOG.error("Can't close container #{}", containerId, e);
    } finally {
        long endTime = Time.monotonicNow();
        totalTime += endTime - startTime;
    }
}

Example 5 with ContainerCommandRequestProto

use of org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto in project ozone by apache.

the class GrpcXceiverService method send.

@Override
public StreamObserver<ContainerCommandRequestProto> send(StreamObserver<ContainerCommandResponseProto> responseObserver) {
    return new StreamObserver<ContainerCommandRequestProto>() {

        private final AtomicBoolean isClosed = new AtomicBoolean(false);

        @Override
        public void onNext(ContainerCommandRequestProto request) {
            try {
                ContainerCommandResponseProto resp = dispatcher.dispatch(request, null);
                responseObserver.onNext(resp);
            } catch (Throwable e) {
                LOG.error("Got exception when processing" + " ContainerCommandRequestProto {}", request, e);
                responseObserver.onError(e);
            }
        }

        @Override
        public void onError(Throwable t) {
            // for now we just log a msg
            LOG.error("ContainerCommand send on error. Exception: ", t);
        }

        @Override
        public void onCompleted() {
            if (isClosed.compareAndSet(false, true)) {
                LOG.debug("ContainerCommand send completed");
                responseObserver.onCompleted();
            }
        }
    };
}