use of org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl in project ozone by apache.
the class TestSCMSafeModeManager method testSafeModePipelineExitRule.
@Test
public void testSafeModePipelineExitRule() throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
String storageDir = GenericTestUtils.getTempPath(TestSCMSafeModeManager.class.getName() + UUID.randomUUID());
try {
MockNodeManager nodeManager = new MockNodeManager(true, 3);
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
// enable pipeline check
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
Pipeline pipeline = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
pipeline = pipelineManager.getPipeline(pipeline.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline);
scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, queue, serviceManager, scmContext);
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers));
assertTrue(scmSafeModeManager.getInSafeMode());
firePipelineEvent(pipelineManager, pipeline);
GenericTestUtils.waitFor(() -> {
return !scmSafeModeManager.getInSafeMode();
}, 100, 1000 * 10);
pipelineManager.close();
} finally {
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, false);
FileUtil.fullyDelete(new File(storageDir));
}
}
use of org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl in project ozone by apache.
the class TestSCMNodeManager method createNodeManager.
/**
* Creates a NodeManager.
*
* @param config - Config for the node manager.
* @return SCNNodeManager
* @throws IOException
*/
SCMNodeManager createNodeManager(OzoneConfiguration config) throws IOException, AuthenticationException {
scm = HddsTestUtils.getScm(config);
scmContext = new SCMContext.Builder().setIsInSafeMode(true).setLeader(true).setIsPreCheckComplete(true).setSCM(scm).build();
PipelineManagerImpl pipelineManager = (PipelineManagerImpl) scm.getPipelineManager();
pipelineManager.setScmContext(scmContext);
return (SCMNodeManager) scm.getScmNodeManager();
}
use of org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl in project ozone by apache.
the class TestHealthyPipelineSafeModeRule method testHealthyPipelineSafeModeRuleWithMixedPipelines.
@Test
public void testHealthyPipelineSafeModeRuleWithMixedPipelines() throws Exception {
String storageDir = GenericTestUtils.getTempPath(TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
EventQueue eventQueue = new EventQueue();
SCMServiceManager serviceManager = new SCMServiceManager();
SCMContext scmContext = SCMContext.emptyContext();
List<ContainerInfo> containers = new ArrayList<>(HddsTestUtils.getContainerInfo(1));
OzoneConfiguration config = new OzoneConfiguration();
// In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are
// stale and last one is dead, and this repeats. So for a 12 node, 9
// healthy, 2 stale and one dead.
MockNodeManager nodeManager = new MockNodeManager(true, 12);
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
// enable pipeline check
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config);
try {
PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
// Create 3 pipelines
Pipeline pipeline1 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.ONE));
pipelineManager.openPipeline(pipeline1.getId());
Pipeline pipeline2 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
pipelineManager.openPipeline(pipeline2.getId());
Pipeline pipeline3 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
pipelineManager.openPipeline(pipeline3.getId());
// Mark pipeline healthy
pipeline1 = pipelineManager.getPipeline(pipeline1.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline1);
pipeline2 = pipelineManager.getPipeline(pipeline2.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline2);
pipeline3 = pipelineManager.getPipeline(pipeline3.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline3);
SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, eventQueue, serviceManager, scmContext);
HealthyPipelineSafeModeRule healthyPipelineSafeModeRule = scmSafeModeManager.getHealthyPipelineSafeModeRule();
// No pipeline event have sent to SCMSafemodeManager
Assert.assertFalse(healthyPipelineSafeModeRule.validate());
GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(LoggerFactory.getLogger(SCMSafeModeManager.class));
// fire event with pipeline create status with ratis type and factor 1
// pipeline, validate() should return false
firePipelineEvent(pipeline1, eventQueue);
GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains("reported count is 1"), 1000, 5000);
Assert.assertFalse(healthyPipelineSafeModeRule.validate());
firePipelineEvent(pipeline2, eventQueue);
firePipelineEvent(pipeline3, eventQueue);
GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000);
} finally {
scmMetadataStore.getStore().close();
FileUtil.fullyDelete(new File(storageDir));
}
}
use of org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl in project ozone by apache.
the class TestSCMSafeModeManager method testPipelinesNotCreatedUntilPreCheckPasses.
@Test
@Ignore("The test is failing, enable after fixing it")
public void testPipelinesNotCreatedUntilPreCheckPasses() throws Exception {
int numOfDns = 5;
// enable pipeline check
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
config.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, numOfDns);
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, true);
MockNodeManager nodeManager = new MockNodeManager(true, numOfDns);
String storageDir = GenericTestUtils.getTempPath(TestSCMSafeModeManager.class.getName() + UUID.randomUUID());
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
// enable pipeline check
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
SafeModeEventHandler smHandler = new SafeModeEventHandler();
queue.addHandler(SCMEvents.SAFE_MODE_STATUS, smHandler);
scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, queue, serviceManager, scmContext);
// Assert SCM is in Safe mode.
assertTrue(scmSafeModeManager.getInSafeMode());
// Register all DataNodes except last one and assert SCM is in safe mode.
for (int i = 0; i < numOfDns - 1; i++) {
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers));
assertTrue(scmSafeModeManager.getInSafeMode());
assertFalse(scmSafeModeManager.getPreCheckComplete());
}
queue.processAll(5000);
Assert.assertEquals(0, smHandler.getInvokedCount());
// Register last DataNode and check that the SafeModeEvent gets fired, but
// safemode is still enabled with preCheck completed.
queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers));
queue.processAll(5000);
Assert.assertEquals(1, smHandler.getInvokedCount());
Assert.assertEquals(true, smHandler.getPreCheckComplete());
Assert.assertEquals(true, smHandler.getIsInSafeMode());
/* There is a race condition where the background pipeline creation
* task creates the pipeline before the following create call.
* So wrapping it with try..catch.
*/
Pipeline pipeline;
try {
pipeline = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
} catch (SCMException ex) {
pipeline = pipelineManager.getPipelines(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)).get(0);
}
// Mark pipeline healthy
pipeline = pipelineManager.getPipeline(pipeline.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline);
firePipelineEvent(pipelineManager, pipeline);
queue.processAll(5000);
Assert.assertEquals(2, smHandler.getInvokedCount());
Assert.assertEquals(true, smHandler.getPreCheckComplete());
Assert.assertEquals(false, smHandler.getIsInSafeMode());
}
use of org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl in project ozone by apache.
the class TestSCMSafeModeManager method testSafeModeExitRuleWithPipelineAvailabilityCheck.
public void testSafeModeExitRuleWithPipelineAvailabilityCheck(int containerCount, int nodeCount, int pipelineCount, double healthyPipelinePercent, double oneReplicaPercent) throws Exception {
OzoneConfiguration conf = createConf(healthyPipelinePercent, oneReplicaPercent);
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(containerCount));
MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount);
PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(conf, MockSCMHAManager.getInstance(true), mockNodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), config);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
pipelineManager.getBackgroundPipelineCreator().stop();
for (int i = 0; i < pipelineCount; i++) {
// Create pipeline
Pipeline pipeline = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
pipelineManager.openPipeline(pipeline.getId());
// Mark pipeline healthy
pipeline = pipelineManager.getPipeline(pipeline.getId());
MockRatisPipelineProvider.markPipelineHealthy(pipeline);
}
for (ContainerInfo container : containers) {
container.setState(HddsProtos.LifeCycleState.CLOSED);
}
scmSafeModeManager = new SCMSafeModeManager(conf, containers, null, pipelineManager, queue, serviceManager, scmContext);
assertTrue(scmSafeModeManager.getInSafeMode());
testContainerThreshold(containers, 1.0);
List<Pipeline> pipelines = pipelineManager.getPipelines();
int healthyPipelineThresholdCount = scmSafeModeManager.getHealthyPipelineSafeModeRule().getHealthyPipelineThresholdCount();
int oneReplicaThresholdCount = scmSafeModeManager.getOneReplicaPipelineSafeModeRule().getThresholdCount();
Assert.assertEquals(healthyPipelineThresholdCount, scmSafeModeManager.getSafeModeMetrics().getNumHealthyPipelinesThreshold().value());
Assert.assertEquals(oneReplicaThresholdCount, scmSafeModeManager.getSafeModeMetrics().getNumPipelinesWithAtleastOneReplicaReportedThreshold().value());
// validate will return true, and add this to validatedRules.
if (Math.max(healthyPipelinePercent, oneReplicaThresholdCount) == 0) {
firePipelineEvent(pipelineManager, pipelines.get(0));
}
for (int i = 0; i < Math.max(healthyPipelineThresholdCount, Math.min(oneReplicaThresholdCount, pipelines.size())); i++) {
firePipelineEvent(pipelineManager, pipelines.get(i));
if (i < healthyPipelineThresholdCount) {
checkHealthy(i + 1);
Assert.assertEquals(i + 1, scmSafeModeManager.getSafeModeMetrics().getCurrentHealthyPipelinesCount().value());
}
if (i < oneReplicaThresholdCount) {
checkOpen(i + 1);
Assert.assertEquals(i + 1, scmSafeModeManager.getSafeModeMetrics().getCurrentPipelinesWithAtleastOneReplicaCount().value());
}
}
Assert.assertEquals(healthyPipelineThresholdCount, scmSafeModeManager.getSafeModeMetrics().getCurrentHealthyPipelinesCount().value());
Assert.assertEquals(oneReplicaThresholdCount, scmSafeModeManager.getSafeModeMetrics().getCurrentPipelinesWithAtleastOneReplicaCount().value());
GenericTestUtils.waitFor(() -> {
return !scmSafeModeManager.getInSafeMode();
}, 100, 1000 * 5);
}
Aggregations