Examples with OzoneContainer org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer used on opensource projects

Example 1 with OzoneContainer

use of org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer in project ozone by apache.

the class CloseContainerCommandHandler method handle.

/**
 * Handles a given SCM command.
 *
 * @param command           - SCM Command
 * @param ozoneContainer    - Ozone Container.
 * @param context           - Current Context.
 * @param connectionManager - The SCMs that we are talking to.
 */
@Override
public void handle(SCMCommand command, OzoneContainer ozoneContainer, StateContext context, SCMConnectionManager connectionManager) {
    invocationCount.incrementAndGet();
    final long startTime = Time.monotonicNow();
    final DatanodeDetails datanodeDetails = context.getParent().getDatanodeDetails();
    final CloseContainerCommandProto closeCommand = ((CloseContainerCommand) command).getProto();
    final ContainerController controller = ozoneContainer.getController();
    final long containerId = closeCommand.getContainerID();
    LOG.debug("Processing Close Container command container #{}", containerId);
    try {
        final Container container = controller.getContainer(containerId);
        if (container == null) {
            LOG.error("Container #{} does not exist in datanode. " + "Container close failed.", containerId);
            return;
        }
        // move the container to CLOSING if in OPEN state
        controller.markContainerForClose(containerId);
        switch(container.getContainerState()) {
            case OPEN:
            case CLOSING:
                // If the container is part of open pipeline, close it via write channel
                if (ozoneContainer.getWriteChannel().isExist(closeCommand.getPipelineID())) {
                    ContainerCommandRequestProto request = getContainerCommandRequestProto(datanodeDetails, closeCommand.getContainerID(), command.getEncodedToken());
                    ozoneContainer.getWriteChannel().submitRequest(request, closeCommand.getPipelineID());
                } else {
                    controller.quasiCloseContainer(containerId);
                    LOG.info("Marking Container {} quasi closed", containerId);
                }
                break;
            case QUASI_CLOSED:
                if (closeCommand.getForce()) {
                    controller.closeContainer(containerId);
                }
                break;
            case CLOSED:
                break;
            case UNHEALTHY:
            case INVALID:
                LOG.debug("Cannot close the container #{}, the container is" + " in {} state.", containerId, container.getContainerState());
                break;
            default:
                break;
        }
    } catch (NotLeaderException e) {
        LOG.debug("Follower cannot close container #{}.", containerId);
    } catch (IOException e) {
        LOG.error("Can't close container #{}", containerId, e);
    } finally {
        long endTime = Time.monotonicNow();
        totalTime += endTime - startTime;
    }
}

Example 2 with OzoneContainer

use of org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer in project ozone by apache.

the class TestEndPoint method testGetVersionAssertRpcTimeOut.

@Test
public /**
 * This test makes a getVersionRPC call, but the DummyStorageServer is
 * going to respond little slowly. We will assert that we are still in the
 * GETVERSION state after the timeout.
 */
void testGetVersionAssertRpcTimeOut() throws Exception {
    final long rpcTimeout = 1000;
    final long tolerance = 100;
    OzoneConfiguration conf = SCMTestUtils.getConf();
    try (EndpointStateMachine rpcEndPoint = createEndpoint(conf, serverAddress, (int) rpcTimeout)) {
        rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
        DatanodeDetails datanodeDetails = randomDatanodeDetails();
        OzoneContainer ozoneContainer = new OzoneContainer(datanodeDetails, conf, getContext(datanodeDetails), null);
        VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint, conf, ozoneContainer);
        scmServerImpl.setRpcResponseDelay(1500);
        long start = Time.monotonicNow();
        EndpointStateMachine.EndPointStates newState = versionTask.call();
        long end = Time.monotonicNow();
        scmServerImpl.setRpcResponseDelay(0);
        Assert.assertThat(end - start, lessThanOrEqualTo(rpcTimeout + tolerance));
        Assert.assertEquals(EndpointStateMachine.EndPointStates.GETVERSION, newState);
    }
}

Example 3 with OzoneContainer

use of org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer in project ozone by apache.

the class TestPipelineClose method testPipelineCloseWithPipelineAction.

@Test
public void testPipelineCloseWithPipelineAction() throws Exception {
    List<DatanodeDetails> dns = ratisContainer.getPipeline().getNodes();
    PipelineActionsFromDatanode pipelineActionsFromDatanode = HddsTestUtils.getPipelineActionFromDatanode(dns.get(0), ratisContainer.getPipeline().getId());
    // send closing action for pipeline
    PipelineActionHandler pipelineActionHandler = new PipelineActionHandler(pipelineManager, SCMContext.emptyContext(), conf);
    pipelineActionHandler.onMessage(pipelineActionsFromDatanode, new EventQueue());
    Thread.sleep(5000);
    OzoneContainer ozoneContainer = cluster.getHddsDatanodes().get(0).getDatanodeStateMachine().getContainer();
    List<PipelineReport> pipelineReports = ozoneContainer.getPipelineReport().getPipelineReportList();
    for (PipelineReport pipelineReport : pipelineReports) {
        // ensure the pipeline is not reported by any dn
        Assert.assertNotEquals(PipelineID.getFromProtobuf(pipelineReport.getPipelineID()), ratisContainer.getPipeline().getId());
    }
    try {
        pipelineManager.getPipeline(ratisContainer.getPipeline().getId());
        Assert.fail("Pipeline should not exist in SCM");
    } catch (PipelineNotFoundException e) {
    }
}

Example 4 with OzoneContainer

use of org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer in project ozone by apache.

the class TestStorageContainerManagerHelper method getContainerMetadata.

private ReferenceCountedDB getContainerMetadata(Long containerID) throws IOException {
    ContainerWithPipeline containerWithPipeline = cluster.getStorageContainerManager().getClientProtocolServer().getContainerWithPipeline(containerID);
    DatanodeDetails dn = containerWithPipeline.getPipeline().getFirstNode();
    OzoneContainer containerServer = getContainerServerByDatanodeUuid(dn.getUuidString());
    KeyValueContainerData containerData = (KeyValueContainerData) containerServer.getContainerSet().getContainer(containerID).getContainerData();
    return BlockUtils.getDB(containerData, conf);
}

Example 5 with OzoneContainer

use of org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer in project ozone by apache.

the class TestValidateBCSIDOnRestart method testValidateBCSIDOnDnRestart.

@Test
public void testValidateBCSIDOnDnRestart() throws Exception {
    OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    // First write and flush creates a container in the datanode
    key.write("ratis".getBytes(UTF_8));
    key.flush();
    key.write("ratis".getBytes(UTF_8));
    KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
    List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList();
    Assert.assertEquals(1, locationInfoList.size());
    OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
    HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, cluster);
    ContainerData containerData = TestHelper.getDatanodeService(omKeyLocationInfo, cluster).getDatanodeStateMachine().getContainer().getContainerSet().getContainer(omKeyLocationInfo.getContainerID()).getContainerData();
    Assert.assertTrue(containerData instanceof KeyValueContainerData);
    KeyValueContainerData keyValueContainerData = (KeyValueContainerData) containerData;
    key.close();
    long containerID = omKeyLocationInfo.getContainerID();
    int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
    // delete the container db file
    FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
    HddsDatanodeService dnService = cluster.getHddsDatanodes().get(index);
    OzoneContainer ozoneContainer = dnService.getDatanodeStateMachine().getContainer();
    ozoneContainer.getContainerSet().removeContainer(containerID);
    ContainerStateMachine stateMachine = (ContainerStateMachine) TestHelper.getStateMachine(cluster.getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
    SimpleStateMachineStorage storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
    stateMachine.takeSnapshot();
    Path parentPath = storage.findLatestSnapshot().getFile().getPath();
    stateMachine.buildMissingContainerSet(parentPath.toFile());
    // Since the snapshot threshold is set to 1, since there are
    // applyTransactions, we should see snapshots
    Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
    // make sure the missing containerSet is not empty
    HddsDispatcher dispatcher = (HddsDispatcher) ozoneContainer.getDispatcher();
    Assert.assertTrue(!dispatcher.getMissingContainerSet().isEmpty());
    Assert.assertTrue(dispatcher.getMissingContainerSet().contains(containerID));
    // write a new key
    key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    // First write and flush creates a container in the datanode
    key.write("ratis1".getBytes(UTF_8));
    key.flush();
    groupOutputStream = (KeyOutputStream) key.getOutputStream();
    locationInfoList = groupOutputStream.getLocationInfoList();
    Assert.assertEquals(1, locationInfoList.size());
    omKeyLocationInfo = locationInfoList.get(0);
    key.close();
    containerID = omKeyLocationInfo.getContainerID();
    dn = TestHelper.getDatanodeService(omKeyLocationInfo, cluster);
    containerData = dn.getDatanodeStateMachine().getContainer().getContainerSet().getContainer(omKeyLocationInfo.getContainerID()).getContainerData();
    Assert.assertTrue(containerData instanceof KeyValueContainerData);
    keyValueContainerData = (KeyValueContainerData) containerData;
    ReferenceCountedDB db = BlockUtils.getDB(keyValueContainerData, conf);
    // modify the bcsid for the container in the ROCKS DB thereby inducing
    // corruption
    db.getStore().getMetadataTable().put(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, 0L);
    db.decrementReference();
    // after the restart, there will be a mismatch in BCSID of what is recorded
    // in the and what is there in RockSDB and hence the container would be
    // marked unhealthy
    index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
    cluster.restartHddsDatanode(dn.getDatanodeDetails(), false);
    // Make sure the container is marked unhealthy
    Assert.assertTrue(cluster.getHddsDatanodes().get(index).getDatanodeStateMachine().getContainer().getContainerSet().getContainer(containerID).getContainerState() == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
}