Search in sources :

Example 11 with MutableVolumeSet

use of org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet in project ozone by apache.

the class TestKeyValueBlockIterator method setUp.

@Before
public void setUp() throws Exception {
    testRoot = GenericTestUtils.getRandomizedTestDir();
    conf = new OzoneConfiguration();
    conf.set(HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testRoot.getAbsolutePath());
    volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    containerData = new KeyValueContainerData(105L, layout, (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), UUID.randomUUID().toString());
    // Init the container.
    container = new KeyValueContainer(containerData, conf);
    container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID.randomUUID().toString());
    db = BlockUtils.getDB(containerData, conf);
}
Also used : RoundRobinVolumeChoosingPolicy(org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) Before(org.junit.Before)

Example 12 with MutableVolumeSet

use of org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet in project ozone by apache.

the class VersionEndpointTask method call.

/**
 * Computes a result, or throws an exception if unable to do so.
 *
 * @return computed result
 * @throws Exception if unable to compute a result
 */
@Override
public EndpointStateMachine.EndPointStates call() throws Exception {
    rpcEndPoint.lock();
    try {
        if (rpcEndPoint.getState().equals(EndpointStateMachine.EndPointStates.GETVERSION)) {
            SCMVersionResponseProto versionResponse = rpcEndPoint.getEndPoint().getVersion(null);
            VersionResponse response = VersionResponse.getFromProtobuf(versionResponse);
            rpcEndPoint.setVersion(response);
            if (!rpcEndPoint.isPassive()) {
                // If end point is passive, datanode does not need to check volumes.
                String scmId = response.getValue(OzoneConsts.SCM_ID);
                String clusterId = response.getValue(OzoneConsts.CLUSTER_ID);
                // Check volumes
                MutableVolumeSet volumeSet = ozoneContainer.getVolumeSet();
                volumeSet.writeLock();
                try {
                    Map<String, StorageVolume> volumeMap = volumeSet.getVolumeMap();
                    Preconditions.checkNotNull(scmId, "Reply from SCM: scmId cannot be null");
                    Preconditions.checkNotNull(clusterId, "Reply from SCM: clusterId cannot be null");
                    // create version file and also set scm ID or cluster ID.
                    for (Map.Entry<String, StorageVolume> entry : volumeMap.entrySet()) {
                        StorageVolume volume = entry.getValue();
                        boolean result = HddsVolumeUtil.checkVolume((HddsVolume) volume, scmId, clusterId, configuration, LOG);
                        if (!result) {
                            volumeSet.failVolume(volume.getStorageDir().getPath());
                        }
                    }
                    if (volumeSet.getVolumesList().size() == 0) {
                        // All volumes are in inconsistent state
                        throw new DiskOutOfSpaceException("All configured Volumes are in Inconsistent State");
                    }
                } finally {
                    volumeSet.writeUnlock();
                }
                // Start the container services after getting the version information
                ozoneContainer.start(clusterId);
            }
            EndpointStateMachine.EndPointStates nextState = rpcEndPoint.getState().getNextState();
            rpcEndPoint.setState(nextState);
            rpcEndPoint.zeroMissedCount();
        } else {
            LOG.debug("Cannot execute GetVersion task as endpoint state machine " + "is in {} state", rpcEndPoint.getState());
        }
    } catch (DiskOutOfSpaceException ex) {
        rpcEndPoint.setState(EndpointStateMachine.EndPointStates.SHUTDOWN);
    } catch (IOException ex) {
        rpcEndPoint.logIfNeeded(ex);
    } finally {
        rpcEndPoint.unlock();
    }
    return rpcEndPoint.getState();
}
Also used : VersionResponse(org.apache.hadoop.ozone.protocol.VersionResponse) IOException(java.io.IOException) DiskOutOfSpaceException(org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException) EndpointStateMachine(org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine) StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) Map(java.util.Map) SCMVersionResponseProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto)

Example 13 with MutableVolumeSet

use of org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet in project ozone by apache.

the class TestContainerMetrics method testContainerMetrics.

@Test
public void testContainerMetrics() throws Exception {
    XceiverServerGrpc server = null;
    XceiverClientGrpc client = null;
    long containerID = ContainerTestHelper.getTestContainerID();
    String path = GenericTestUtils.getRandomizedTempPath();
    try {
        final int interval = 1;
        Pipeline pipeline = MockPipeline.createSingleNodePipeline();
        OzoneConfiguration conf = new OzoneConfiguration();
        conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
        conf.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, interval);
        DatanodeDetails datanodeDetails = randomDatanodeDetails();
        conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path);
        conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path);
        VolumeSet volumeSet = new MutableVolumeSet(datanodeDetails.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
        ContainerSet containerSet = new ContainerSet();
        DatanodeStateMachine stateMachine = Mockito.mock(DatanodeStateMachine.class);
        StateContext context = Mockito.mock(StateContext.class);
        Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
        Mockito.when(context.getParent()).thenReturn(stateMachine);
        ContainerMetrics metrics = ContainerMetrics.create(conf);
        Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
        for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) {
            handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, context.getParent().getDatanodeDetails().getUuidString(), containerSet, volumeSet, metrics, c -> {
            }));
        }
        HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, null);
        dispatcher.setClusterId(UUID.randomUUID().toString());
        server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher, null);
        client = new XceiverClientGrpc(pipeline, conf);
        server.start();
        client.connect();
        // Create container
        ContainerCommandRequestProto request = ContainerTestHelper.getCreateContainerRequest(containerID, pipeline);
        ContainerCommandResponseProto response = client.sendCommand(request);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
        // Write Chunk
        BlockID blockID = ContainerTestHelper.getTestBlockID(containerID);
        ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, 1024, null);
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, 1024, null);
        response = client.sendCommand(writeChunkRequest);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
        // Read Chunk
        ContainerProtos.ContainerCommandRequestProto readChunkRequest = ContainerTestHelper.getReadChunkRequest(pipeline, writeChunkRequest.getWriteChunk());
        response = client.sendCommand(readChunkRequest);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
        MetricsRecordBuilder containerMetrics = getMetrics("StorageContainerMetrics");
        assertCounter("NumOps", 3L, containerMetrics);
        assertCounter("numCreateContainer", 1L, containerMetrics);
        assertCounter("numWriteChunk", 1L, containerMetrics);
        assertCounter("numReadChunk", 1L, containerMetrics);
        assertCounter("bytesWriteChunk", 1024L, containerMetrics);
        assertCounter("bytesReadChunk", 1024L, containerMetrics);
        String sec = interval + "s";
        Thread.sleep((interval + 1) * 1000);
        assertQuantileGauges("WriteChunkNanos" + sec, containerMetrics);
        // Check VolumeIOStats metrics
        List<HddsVolume> volumes = StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList());
        HddsVolume hddsVolume = volumes.get(0);
        MetricsRecordBuilder volumeIOMetrics = getMetrics(hddsVolume.getVolumeIOStats().getMetricsSourceName());
        assertCounter("ReadBytes", 1024L, volumeIOMetrics);
        assertCounter("ReadOpCount", 1L, volumeIOMetrics);
        assertCounter("WriteBytes", 1024L, volumeIOMetrics);
        assertCounter("WriteOpCount", 1L, volumeIOMetrics);
    } finally {
        if (client != null) {
            client.close();
        }
        if (server != null) {
            server.stop();
        }
        // clean up volume dir
        File file = new File(path);
        if (file.exists()) {
            FileUtil.fullyDelete(file);
        }
    }
}
Also used : ScmConfigKeys(org.apache.hadoop.hdds.scm.ScmConfigKeys) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MockPipeline(org.apache.hadoop.hdds.scm.pipeline.MockPipeline) MockDatanodeDetails.randomDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails) BlockID(org.apache.hadoop.hdds.client.BlockID) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) MetricsAsserts.getMetrics(org.apache.hadoop.test.MetricsAsserts.getMetrics) HddsVolume(org.apache.hadoop.ozone.container.common.volume.HddsVolume) MetricsAsserts.assertCounter(org.apache.hadoop.test.MetricsAsserts.assertCounter) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) MetricsRecordBuilder(org.apache.hadoop.metrics2.MetricsRecordBuilder) Map(java.util.Map) Timeout(org.junit.rules.Timeout) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) StorageVolumeUtil(org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil) FileUtil(org.apache.hadoop.fs.FileUtil) ContainerTestHelper(org.apache.hadoop.ozone.container.ContainerTestHelper) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test) UUID(java.util.UUID) XceiverServerGrpc(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc) Maps(com.google.common.collect.Maps) File(java.io.File) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) Mockito(org.mockito.Mockito) List(java.util.List) StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet) Rule(org.junit.Rule) MetricsAsserts.assertQuantileGauges(org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges) OzoneConfigKeys(org.apache.hadoop.ozone.OzoneConfigKeys) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) Assert(org.junit.Assert) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) DFSConfigKeysLegacy(org.apache.hadoop.hdds.DFSConfigKeysLegacy) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto) MockDatanodeDetails.randomDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet) MetricsRecordBuilder(org.apache.hadoop.metrics2.MetricsRecordBuilder) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) XceiverServerGrpc(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc) MockPipeline(org.apache.hadoop.hdds.scm.pipeline.MockPipeline) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) HddsVolume(org.apache.hadoop.ozone.container.common.volume.HddsVolume) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) File(java.io.File) Test(org.junit.Test)

Example 14 with MutableVolumeSet

use of org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet in project ozone by apache.

the class TestSecureContainerServer method createDispatcher.

private static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, OzoneConfiguration conf) throws IOException {
    ContainerSet containerSet = new ContainerSet();
    conf.set(HDDS_DATANODE_DIR_KEY, Paths.get(TEST_DIR, "dfs", "data", "hdds", RandomStringUtils.randomAlphabetic(4)).toString());
    conf.set(OZONE_METADATA_DIRS, TEST_DIR);
    VolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    DatanodeStateMachine stateMachine = Mockito.mock(DatanodeStateMachine.class);
    StateContext context = Mockito.mock(StateContext.class);
    Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd);
    Mockito.when(context.getParent()).thenReturn(stateMachine);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
    for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) {
        handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, dd.getUuid().toString(), containerSet, volumeSet, metrics, c -> {
        }));
    }
    HddsDispatcher hddsDispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, TokenVerifier.create(new SecurityConfig((conf)), caClient));
    hddsDispatcher.setClusterId(scmId.toString());
    return hddsDispatcher;
}
Also used : MockPipeline(org.apache.hadoop.hdds.scm.pipeline.MockPipeline) XceiverServerSpi(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi) ContainerTestHelper.newGetBlockRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newGetBlockRequestBuilder) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto) ContainerProtocolCalls(org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) DefaultMetricsSystem(org.apache.hadoop.metrics2.lib.DefaultMetricsSystem) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) Map(java.util.Map) After(org.junit.After) MockDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails) HddsConfigKeys(org.apache.hadoop.hdds.HddsConfigKeys) CheckedBiConsumer(org.apache.ratis.util.function.CheckedBiConsumer) XceiverServerRatis(org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis) EnumSet(java.util.EnumSet) OZONE_METADATA_DIRS(org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS) CertificateClientTestImpl(org.apache.hadoop.ozone.client.CertificateClientTestImpl) AfterClass(org.junit.AfterClass) SecurityConfig(org.apache.hadoop.hdds.security.x509.SecurityConfig) TokenVerifier(org.apache.hadoop.hdds.security.token.TokenVerifier) ContainerTestHelper.getCreateContainerRequest(org.apache.hadoop.ozone.container.ContainerTestHelper.getCreateContainerRequest) OZONE_SECURITY_ENABLED_KEY(org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY) UUID(java.util.UUID) XceiverServerGrpc(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc) List(java.util.List) OzoneBlockTokenSecretManager(org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager) XceiverClientSpi(org.apache.hadoop.hdds.scm.XceiverClientSpi) HDDS_BLOCK_TOKEN_ENABLED(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED) OzoneConfigKeys(org.apache.hadoop.ozone.OzoneConfigKeys) RandomStringUtils(org.apache.commons.lang3.RandomStringUtils) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) ContainerTestHelper.getTestBlockID(org.apache.hadoop.ozone.container.ContainerTestHelper.getTestBlockID) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) RandomUtils(org.apache.commons.lang3.RandomUtils) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) RatisTestHelper(org.apache.hadoop.ozone.RatisTestHelper) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerController(org.apache.hadoop.ozone.container.ozoneimpl.ContainerController) BeforeClass(org.junit.BeforeClass) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) ContainerTestHelper.newReadChunkRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newReadChunkRequestBuilder) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) ArrayList(java.util.ArrayList) XceiverClientRatis(org.apache.hadoop.hdds.scm.XceiverClientRatis) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) ContainerTestHelper.getTestContainerID(org.apache.hadoop.ozone.container.ContainerTestHelper.getTestContainerID) ContainerDispatcher(org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) ContainerTestHelper.newGetCommittedBlockLengthBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newGetCommittedBlockLengthBuilder) HddsUtils.isReadOnly(org.apache.hadoop.hdds.HddsUtils.isReadOnly) AccessModeProto(org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto) Assert.assertNotNull(org.junit.Assert.assertNotNull) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Token(org.apache.hadoop.security.token.Token) Maps(com.google.common.collect.Maps) File(java.io.File) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerTokenSecretManager(org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager) ContainerTestHelper.newPutBlockRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newPutBlockRequestBuilder) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Mockito(org.mockito.Mockito) OzoneBlockTokenIdentifier(org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier) StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet) Paths(java.nio.file.Paths) HDDS_DATANODE_DIR_KEY(org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY) ContainerTestHelper.newWriteChunkRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newWriteChunkRequestBuilder) RpcType(org.apache.ratis.rpc.RpcType) ExitUtils(org.apache.ratis.util.ExitUtils) GRPC(org.apache.ratis.rpc.SupportedRpcType.GRPC) Assert(org.junit.Assert) SUCCESS(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.SUCCESS) Assert.assertEquals(org.junit.Assert.assertEquals) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) SecurityConfig(org.apache.hadoop.hdds.security.x509.SecurityConfig) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet)

Example 15 with MutableVolumeSet

use of org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet in project ozone by apache.

the class TestDatanodeHddsVolumeFailureToleration method testDNCorrectlyHandlesVolumeFailureOnStartup.

@Test
public void testDNCorrectlyHandlesVolumeFailureOnStartup() throws Exception {
    HddsDatanodeService dn = datanodes.get(0);
    OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
    MutableVolumeSet volSet = oc.getVolumeSet();
    StorageVolume vol0 = volSet.getVolumesList().get(0);
    StorageVolume vol1 = volSet.getVolumesList().get(1);
    File volRootDir0 = vol0.getStorageDir();
    File volRootDir1 = vol1.getStorageDir();
    // simulate bad volumes <= tolerated
    DatanodeTestUtils.simulateBadRootDir(volRootDir0);
    // restart datanode to test
    cluster.restartHddsDatanode(0, true);
    // no exception is good
    // fail a second volume
    DatanodeTestUtils.simulateBadRootDir(volRootDir1);
    // restart datanode to test
    try {
        cluster.restartHddsDatanode(0, true);
        Assert.fail();
    } catch (RuntimeException e) {
        Assert.assertTrue(e.getMessage().contains("Can't start the HDDS datanode plugin"));
    }
    // restore bad volumes
    DatanodeTestUtils.restoreBadRootDir(volRootDir0);
    DatanodeTestUtils.restoreBadRootDir(volRootDir1);
}
Also used : StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) HddsDatanodeService(org.apache.hadoop.ozone.HddsDatanodeService) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) OzoneContainer(org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer) File(java.io.File) Test(org.junit.Test)

Aggregations

MutableVolumeSet (org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet)25 File (java.io.File)12 Test (org.junit.Test)12 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)10 StorageVolume (org.apache.hadoop.ozone.container.common.volume.StorageVolume)10 ContainerMetrics (org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics)9 ContainerSet (org.apache.hadoop.ozone.container.common.impl.ContainerSet)8 IOException (java.io.IOException)7 HddsVolume (org.apache.hadoop.ozone.container.common.volume.HddsVolume)7 RoundRobinVolumeChoosingPolicy (org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy)7 Handler (org.apache.hadoop.ozone.container.common.interfaces.Handler)6 VolumeSet (org.apache.hadoop.ozone.container.common.volume.VolumeSet)6 DatanodeStateMachine (org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine)5 StateContext (org.apache.hadoop.ozone.container.common.statemachine.StateContext)5 Map (java.util.Map)4 UUID (java.util.UUID)4 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)4 HddsDatanodeService (org.apache.hadoop.ozone.HddsDatanodeService)4 KeyValueContainer (org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer)4 KeyValueContainerData (org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData)4