Search in sources :

Example 11 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestRatisPipelineProvider method init.

public void init(int maxPipelinePerNode, OzoneConfiguration conf) throws Exception {
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    nodeManager = new MockNodeManager(true, 10);
    nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode);
    SCMHAManager scmhaManager = MockSCMHAManager.getInstance(true);
    conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, maxPipelinePerNode);
    stateManager = PipelineStateManagerImpl.newBuilder().setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)).setRatisServer(scmhaManager.getRatisServer()).setNodeManager(nodeManager).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    provider = new MockRatisPipelineProvider(nodeManager, stateManager, conf);
}
Also used : MockSCMHAManager(org.apache.hadoop.hdds.scm.ha.MockSCMHAManager) SCMHAManager(org.apache.hadoop.hdds.scm.ha.SCMHAManager) SCMDBDefinition(org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition) TestContainerManagerImpl(org.apache.hadoop.hdds.scm.container.TestContainerManagerImpl) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager)

Example 12 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelineManagerImpl method init.

@Before
public void init() throws Exception {
    conf = SCMTestUtils.getConf();
    testDir = GenericTestUtils.getTestDir(TestPipelineManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, GenericTestUtils.getRandomizedTempPath());
    scm = HddsTestUtils.getScm(conf);
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    nodeManager = new MockNodeManager(true, 20);
    maxPipelineCount = nodeManager.getNodeCount(HddsProtos.NodeOperationalState.IN_SERVICE, HddsProtos.NodeState.HEALTHY) * conf.getInt(OZONE_DATANODE_PIPELINE_LIMIT, OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT) / HddsProtos.ReplicationFactor.THREE.getNumber();
    scmContext = new SCMContext.Builder().setIsInSafeMode(true).setLeader(true).setIsPreCheckComplete(true).setSCM(scm).build();
    serviceManager = new SCMServiceManager();
}
Also used : SCMDBDefinition(org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition) MetricsRecordBuilder(org.apache.hadoop.metrics2.MetricsRecordBuilder) DBStoreBuilder(org.apache.hadoop.hdds.utils.db.DBStoreBuilder) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) Before(org.junit.Before)

Example 13 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelineStateManagerImpl method init.

@Before
public void init() throws Exception {
    final OzoneConfiguration conf = SCMTestUtils.getConf();
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    SCMHAManager scmhaManager = MockSCMHAManager.getInstance(true);
    NodeManager nodeManager = new MockNodeManager(true, 10);
    stateManager = PipelineStateManagerImpl.newBuilder().setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)).setRatisServer(scmhaManager.getRatisServer()).setNodeManager(nodeManager).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
}
Also used : MockSCMHAManager(org.apache.hadoop.hdds.scm.ha.MockSCMHAManager) SCMHAManager(org.apache.hadoop.hdds.scm.ha.SCMHAManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) SCMDBDefinition(org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) TestContainerManagerImpl(org.apache.hadoop.hdds.scm.container.TestContainerManagerImpl) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) Before(org.junit.Before)

Example 14 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestHealthyPipelineSafeModeRule method testHealthyPipelineSafeModeRuleWithMixedPipelines.

@Test
public void testHealthyPipelineSafeModeRuleWithMixedPipelines() throws Exception {
    String storageDir = GenericTestUtils.getTempPath(TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
    EventQueue eventQueue = new EventQueue();
    SCMServiceManager serviceManager = new SCMServiceManager();
    SCMContext scmContext = SCMContext.emptyContext();
    List<ContainerInfo> containers = new ArrayList<>(HddsTestUtils.getContainerInfo(1));
    OzoneConfiguration config = new OzoneConfiguration();
    // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are
    // stale and last one is dead, and this repeats. So for a 12 node, 9
    // healthy, 2 stale and one dead.
    MockNodeManager nodeManager = new MockNodeManager(true, 12);
    config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
    // enable pipeline check
    config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
    config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config);
    try {
        PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
        PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
        pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
        // Create 3 pipelines
        Pipeline pipeline1 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.ONE));
        pipelineManager.openPipeline(pipeline1.getId());
        Pipeline pipeline2 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
        pipelineManager.openPipeline(pipeline2.getId());
        Pipeline pipeline3 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
        pipelineManager.openPipeline(pipeline3.getId());
        // Mark pipeline healthy
        pipeline1 = pipelineManager.getPipeline(pipeline1.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline1);
        pipeline2 = pipelineManager.getPipeline(pipeline2.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline2);
        pipeline3 = pipelineManager.getPipeline(pipeline3.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline3);
        SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, eventQueue, serviceManager, scmContext);
        HealthyPipelineSafeModeRule healthyPipelineSafeModeRule = scmSafeModeManager.getHealthyPipelineSafeModeRule();
        // No pipeline event have sent to SCMSafemodeManager
        Assert.assertFalse(healthyPipelineSafeModeRule.validate());
        GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(LoggerFactory.getLogger(SCMSafeModeManager.class));
        // fire event with pipeline create status with ratis type and factor 1
        // pipeline, validate() should return false
        firePipelineEvent(pipeline1, eventQueue);
        GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains("reported count is 1"), 1000, 5000);
        Assert.assertFalse(healthyPipelineSafeModeRule.validate());
        firePipelineEvent(pipeline2, eventQueue);
        firePipelineEvent(pipeline3, eventQueue);
        GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000);
    } finally {
        scmMetadataStore.getStore().close();
        FileUtil.fullyDelete(new File(storageDir));
    }
}
Also used : ArrayList(java.util.ArrayList) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) PipelineProvider(org.apache.hadoop.hdds.scm.pipeline.PipelineProvider) EventQueue(org.apache.hadoop.hdds.server.events.EventQueue) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) SCMContext(org.apache.hadoop.hdds.scm.ha.SCMContext) SCMMetadataStoreImpl(org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl) PipelineManagerImpl(org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) SCMMetadataStore(org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore) File(java.io.File) Test(org.junit.Test)

Example 15 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestOneReplicaPipelineSafeModeRule method setup.

private void setup(int nodes, int pipelineFactorThreeCount, int pipelineFactorOneCount) throws Exception {
    OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
    ozoneConfiguration.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
    ozoneConfiguration.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().toString());
    ozoneConfiguration.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    List<ContainerInfo> containers = new ArrayList<>();
    containers.addAll(HddsTestUtils.getContainerInfo(1));
    mockNodeManager = new MockNodeManager(true, nodes);
    eventQueue = new EventQueue();
    serviceManager = new SCMServiceManager();
    scmContext = SCMContext.emptyContext();
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(ozoneConfiguration);
    pipelineManager = PipelineManagerImpl.newPipelineManager(ozoneConfiguration, MockSCMHAManager.getInstance(true), mockNodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), ozoneConfiguration);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
    createPipelines(pipelineFactorThreeCount, HddsProtos.ReplicationFactor.THREE);
    createPipelines(pipelineFactorOneCount, HddsProtos.ReplicationFactor.ONE);
    SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(ozoneConfiguration, containers, null, pipelineManager, eventQueue, serviceManager, scmContext);
    rule = scmSafeModeManager.getOneReplicaPipelineSafeModeRule();
}
Also used : SCMMetadataStoreImpl(org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) ArrayList(java.util.ArrayList) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) SCMMetadataStore(org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) PipelineProvider(org.apache.hadoop.hdds.scm.pipeline.PipelineProvider) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) EventQueue(org.apache.hadoop.hdds.server.events.EventQueue)

Aggregations

MockNodeManager (org.apache.hadoop.hdds.scm.container.MockNodeManager)26 Test (org.junit.Test)14 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)13 ArrayList (java.util.ArrayList)9 MockRatisPipelineProvider (org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider)8 PipelineProvider (org.apache.hadoop.hdds.scm.pipeline.PipelineProvider)8 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)7 Before (org.junit.Before)7 MockDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails)6 SCMServiceManager (org.apache.hadoop.hdds.scm.ha.SCMServiceManager)6 SCMDBDefinition (org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition)6 PipelineManagerImpl (org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl)6 ContainerInfo (org.apache.hadoop.hdds.scm.container.ContainerInfo)5 TestContainerManagerImpl (org.apache.hadoop.hdds.scm.container.TestContainerManagerImpl)5 SCMMetadataStoreImpl (org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl)5 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)5 EventQueue (org.apache.hadoop.hdds.server.events.EventQueue)5 File (java.io.File)4 SCMException (org.apache.hadoop.hdds.scm.exceptions.SCMException)4 MockSCMHAManager (org.apache.hadoop.hdds.scm.ha.MockSCMHAManager)4