Examples with MockNodeManager org.apache.hadoop.hdds.scm.container.MockNodeManager used on opensource projects

Example 1 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestContainerPlacement method testCapacityPlacementYieldsBetterDataDistribution.

/**
 * This test simulates lots of Cluster I/O and updates the metadata in SCM.
 * We simulate adding and removing containers from the cluster. It asserts
 * that our placement algorithm has taken the capacity of nodes into
 * consideration by asserting that standard deviation of used space on these
 * has improved.
 */
@Test
public void testCapacityPlacementYieldsBetterDataDistribution() throws SCMException {
    final int opsCount = 200 * 1000;
    final int nodesRequired = 3;
    Random random = new Random();
    // The nature of init code in MockNodeManager yields similar clusters.
    MockNodeManager nodeManagerCapacity = new MockNodeManager(true, 100);
    MockNodeManager nodeManagerRandom = new MockNodeManager(true, 100);
    DescriptiveStatistics beforeCapacity = computeStatistics(nodeManagerCapacity);
    DescriptiveStatistics beforeRandom = computeStatistics(nodeManagerRandom);
    // Assert that our initial layout of clusters are similar.
    assertEquals(beforeCapacity.getStandardDeviation(), beforeRandom.getStandardDeviation(), 0.001);
    SCMContainerPlacementCapacity capacityPlacer = new SCMContainerPlacementCapacity(nodeManagerCapacity, new OzoneConfiguration(), null, true, null);
    SCMContainerPlacementRandom randomPlacer = new SCMContainerPlacementRandom(nodeManagerRandom, new OzoneConfiguration(), null, true, null);
    for (int x = 0; x < opsCount; x++) {
        long containerSize = random.nextInt(10) * OzoneConsts.GB;
        long metadataSize = random.nextInt(10) * OzoneConsts.GB;
        List<DatanodeDetails> nodesCapacity = capacityPlacer.chooseDatanodes(new ArrayList<>(), null, nodesRequired, metadataSize, containerSize);
        assertEquals(nodesRequired, nodesCapacity.size());
        List<DatanodeDetails> nodesRandom = randomPlacer.chooseDatanodes(nodesCapacity, null, nodesRequired, metadataSize, containerSize);
        // One fifth of all calls are delete
        if (x % 5 == 0) {
            deleteContainer(nodeManagerCapacity, nodesCapacity, containerSize);
            deleteContainer(nodeManagerRandom, nodesRandom, containerSize);
        } else {
            createContainer(nodeManagerCapacity, nodesCapacity, containerSize);
            createContainer(nodeManagerRandom, nodesRandom, containerSize);
        }
    }
    DescriptiveStatistics postCapacity = computeStatistics(nodeManagerCapacity);
    DescriptiveStatistics postRandom = computeStatistics(nodeManagerRandom);
    // This is a very bold claim, and needs large number of I/O operations.
    // The claim in this assertion is that we improved the data distribution
    // of this cluster in relation to the start state of the cluster.
    Assert.assertTrue(beforeCapacity.getStandardDeviation() > postCapacity.getStandardDeviation());
    // This asserts that Capacity placement yields a better placement
    // algorithm than random placement, since both cluster started at an
    // identical state.
    Assert.assertTrue(postRandom.getStandardDeviation() > postCapacity.getStandardDeviation());
}

Example 2 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestKeyManagerImpl method setUp.

@BeforeClass
public static void setUp() throws Exception {
    conf = new OzoneConfiguration();
    dir = GenericTestUtils.getRandomizedTestDir();
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
    conf.set(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, "true");
    mockScmBlockLocationProtocol = mock(ScmBlockLocationProtocol.class);
    nodeManager = new MockNodeManager(true, 10);
    NodeSchema[] schemas = new NodeSchema[] { ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA };
    NodeSchemaManager schemaManager = NodeSchemaManager.getInstance();
    schemaManager.init(schemas, false);
    NetworkTopology clusterMap = new NetworkTopologyImpl(schemaManager);
    nodeManager.getAllNodes().stream().forEach(node -> {
        node.setNetworkName(node.getUuidString());
        clusterMap.add(node);
    });
    ((MockNodeManager) nodeManager).setNetworkTopology(clusterMap);
    SCMConfigurator configurator = new SCMConfigurator();
    configurator.setScmNodeManager(nodeManager);
    configurator.setNetworkTopology(clusterMap);
    configurator.setSCMHAManager(MockSCMHAManager.getInstance(true));
    configurator.setScmContext(SCMContext.emptyContext());
    scm = HddsTestUtils.getScm(conf, configurator);
    scm.start();
    scm.exitSafeMode();
    scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
    conf.setLong(OZONE_KEY_PREALLOCATION_BLOCKS_MAX, 10);
    mockScmContainerClient = Mockito.mock(StorageContainerLocationProtocol.class);
    OmTestManagers omTestManagers = new OmTestManagers(conf, scm.getBlockProtocolServer(), mockScmContainerClient);
    om = omTestManagers.getOzoneManager();
    metadataManager = omTestManagers.getMetadataManager();
    keyManager = (KeyManagerImpl) omTestManagers.getKeyManager();
    prefixManager = omTestManagers.getPrefixManager();
    writeClient = omTestManagers.getWriteClient();
    mockContainerClient();
    Mockito.when(mockScmBlockLocationProtocol.allocateBlock(Mockito.anyLong(), Mockito.anyInt(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow(new SCMException("SafeModePrecheck failed for allocateBlock", ResultCodes.SAFE_MODE_EXCEPTION));
    createVolume(VOLUME_NAME);
    createBucket(VOLUME_NAME, BUCKET_NAME, false);
    createBucket(VOLUME_NAME, VERSIONED_BUCKET_NAME, true);
}

Example 3 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestSCMSafeModeManager method testFailWithIncorrectValueForHealthyPipelinePercent.

@Test
public void testFailWithIncorrectValueForHealthyPipelinePercent() throws Exception {
    try {
        OzoneConfiguration conf = createConf(100, 0.9);
        MockNodeManager mockNodeManager = new MockNodeManager(true, 10);
        PipelineManager pipelineManager = PipelineManagerImpl.newPipelineManager(conf, MockSCMHAManager.getInstance(true), mockNodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
        scmSafeModeManager = new SCMSafeModeManager(conf, containers, null, pipelineManager, queue, serviceManager, scmContext);
        fail("testFailWithIncorrectValueForHealthyPipelinePercent");
    } catch (IllegalArgumentException ex) {
        GenericTestUtils.assertExceptionContains("value should be >= 0.0 and <=" + " 1.0", ex);
    }
}

Example 4 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestSCMSafeModeManager method testSafeModePipelineExitRule.

@Test
public void testSafeModePipelineExitRule() throws Exception {
    containers = new ArrayList<>();
    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
    String storageDir = GenericTestUtils.getTempPath(TestSCMSafeModeManager.class.getName() + UUID.randomUUID());
    try {
        MockNodeManager nodeManager = new MockNodeManager(true, 3);
        config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
        // enable pipeline check
        config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
        PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
        PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
        pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
        Pipeline pipeline = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
        pipeline = pipelineManager.getPipeline(pipeline.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline);
        scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, queue, serviceManager, scmContext);
        queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, HddsTestUtils.createNodeRegistrationContainerReport(containers));
        assertTrue(scmSafeModeManager.getInSafeMode());
        firePipelineEvent(pipelineManager, pipeline);
        GenericTestUtils.waitFor(() -> {
            return !scmSafeModeManager.getInSafeMode();
        }, 100, 1000 * 10);
        pipelineManager.close();
    } finally {
        config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, false);
        FileUtil.fullyDelete(new File(storageDir));
    }
}

Example 5 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestSCMSafeModeManager method testFailWithIncorrectValueForOneReplicaPipelinePercent.

@Test
public void testFailWithIncorrectValueForOneReplicaPipelinePercent() throws Exception {
    try {
        OzoneConfiguration conf = createConf(0.9, 200);
        MockNodeManager mockNodeManager = new MockNodeManager(true, 10);
        PipelineManager pipelineManager = PipelineManagerImpl.newPipelineManager(conf, MockSCMHAManager.getInstance(true), mockNodeManager, scmMetadataStore.getPipelineTable(), queue, scmContext, serviceManager);
        scmSafeModeManager = new SCMSafeModeManager(conf, containers, null, pipelineManager, queue, serviceManager, scmContext);
        fail("testFailWithIncorrectValueForOneReplicaPipelinePercent");
    } catch (IllegalArgumentException ex) {
        GenericTestUtils.assertExceptionContains("value should be >= 0.0 and <=" + " 1.0", ex);
    }
}