Examples with NodeManager org.apache.hadoop.hdds.scm.node.NodeManager used on opensource projects

Example 21 with NodeManager

use of org.apache.hadoop.hdds.scm.node.NodeManager in project ozone by apache.

the class TestReconPipelineManager method testInitialize.

@Test
public void testInitialize() throws IOException {
    // Get 3 OPEN pipelines from SCM.
    List<Pipeline> pipelinesFromScm = getPipelines(3);
    // Recon has 2 pipelines in ALLOCATED state. (1 is valid and 1 is obsolete)
    // Valid pipeline in Allocated state.
    Pipeline validPipeline = Pipeline.newBuilder().setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE)).setId(pipelinesFromScm.get(0).getId()).setNodes(pipelinesFromScm.get(0).getNodes()).setState(Pipeline.PipelineState.ALLOCATED).build();
    // Invalid pipeline.
    Pipeline invalidPipeline = Pipeline.newBuilder().setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE)).setId(PipelineID.randomId()).setNodes(Collections.singletonList(randomDatanodeDetails())).setState(Pipeline.PipelineState.CLOSED).build();
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    EventQueue eventQueue = new EventQueue();
    this.versionManager = Mockito.mock(HDDSLayoutVersionManager.class);
    Mockito.when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    Mockito.when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager);
    try (ReconPipelineManager reconPipelineManager = ReconPipelineManager.newReconPipelineManager(conf, nodeManager, ReconSCMDBDefinition.PIPELINES.getTable(store), eventQueue, scmhaManager, scmContext)) {
        scmContext = new SCMContext.Builder().setIsInSafeMode(true).setLeader(true).setIsPreCheckComplete(true).setSCM(mock(StorageContainerManager.class)).build();
        reconPipelineManager.setScmContext(scmContext);
        reconPipelineManager.addPipeline(validPipeline);
        reconPipelineManager.addPipeline(invalidPipeline);
        reconPipelineManager.initializePipelines(pipelinesFromScm);
        List<Pipeline> newReconPipelines = reconPipelineManager.getPipelines();
        // Test if the number of pipelines in SCM is as expected.
        assertEquals(3, newReconPipelines.size());
        // Test if new pipelines from SCM are picked up.
        for (Pipeline pipeline : pipelinesFromScm) {
            assertTrue(reconPipelineManager.containsPipeline(pipeline.getId()));
        }
        // Test if existing pipeline state is updated.
        assertEquals(Pipeline.PipelineState.OPEN, reconPipelineManager.getPipeline(validPipeline.getId()).getPipelineState());
        // Test if obsolete pipelines in Recon are removed.
        assertFalse(reconPipelineManager.containsPipeline(invalidPipeline.getId()));
    }
}

Example 22 with NodeManager

use of org.apache.hadoop.hdds.scm.node.NodeManager in project ozone by apache.

the class TestReconPipelineManager method testAddPipeline.

@Test
public void testAddPipeline() throws IOException {
    Pipeline pipeline = getRandomPipeline();
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    EventQueue eventQueue = new EventQueue();
    this.versionManager = Mockito.mock(HDDSLayoutVersionManager.class);
    Mockito.when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    Mockito.when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager);
    ReconPipelineManager reconPipelineManager = ReconPipelineManager.newReconPipelineManager(conf, nodeManager, ReconSCMDBDefinition.PIPELINES.getTable(store), eventQueue, scmhaManager, scmContext);
    assertFalse(reconPipelineManager.containsPipeline(pipeline.getId()));
    reconPipelineManager.addPipeline(pipeline);
    assertTrue(reconPipelineManager.containsPipeline(pipeline.getId()));
}

Example 23 with NodeManager

use of org.apache.hadoop.hdds.scm.node.NodeManager in project ozone by apache.

the class TestReconPipelineManager method testStubbedReconPipelineFactory.

@Test
public void testStubbedReconPipelineFactory() throws IOException {
    NodeManager nodeManagerMock = mock(NodeManager.class);
    ReconPipelineManager reconPipelineManager = ReconPipelineManager.newReconPipelineManager(conf, nodeManagerMock, ReconSCMDBDefinition.PIPELINES.getTable(store), new EventQueue(), scmhaManager, scmContext);
    PipelineFactory pipelineFactory = reconPipelineManager.getPipelineFactory();
    assertTrue(pipelineFactory instanceof ReconPipelineFactory);
    ReconPipelineFactory reconPipelineFactory = (ReconPipelineFactory) pipelineFactory;
    assertTrue(reconPipelineFactory.getProviders().isEmpty());
    for (ReplicationType type : reconPipelineFactory.getProviders().keySet()) {
        PipelineProvider pipelineProvider = reconPipelineFactory.getProviders().get(type);
        assertTrue(pipelineProvider instanceof ReconPipelineProvider);
    }
}

Example 24 with NodeManager

use of org.apache.hadoop.hdds.scm.node.NodeManager in project ozone by apache.

the class ContainerPlacementPolicyFactory method getPolicyInternal.

private static PlacementPolicy getPolicyInternal(Class<? extends PlacementPolicy> placementClass, ConfigurationSource conf, final NodeManager nodeManager, NetworkTopology clusterMap, final boolean fallback, SCMContainerPlacementMetrics metrics) throws SCMException {
    Constructor<? extends PlacementPolicy> constructor;
    try {
        constructor = placementClass.getDeclaredConstructor(NodeManager.class, ConfigurationSource.class, NetworkTopology.class, boolean.class, SCMContainerPlacementMetrics.class);
        LOG.info("Create container placement policy of type {}", placementClass.getCanonicalName());
    } catch (NoSuchMethodException e) {
        String msg = "Failed to find constructor(NodeManager, Configuration, " + "NetworkTopology, boolean) for class " + placementClass.getCanonicalName();
        LOG.error(msg);
        throw new SCMException(msg, SCMException.ResultCodes.FAILED_TO_INIT_CONTAINER_PLACEMENT_POLICY);
    }
    try {
        return constructor.newInstance(nodeManager, conf, clusterMap, fallback, metrics);
    } catch (Exception e) {
        throw new RuntimeException("Failed to instantiate class " + placementClass.getCanonicalName() + " for " + e.getMessage());
    }
}

Example 25 with NodeManager

use of org.apache.hadoop.hdds.scm.node.NodeManager in project ozone by apache.

the class TestSCMDatanodeHeartbeatDispatcher method testNodeReportDispatcher.

@Test
public void testNodeReportDispatcher() throws IOException {
    AtomicInteger eventReceived = new AtomicInteger();
    NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
    NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
    Mockito.when(mockNodeManager.isNodeRegistered(Mockito.any())).thenReturn(true);
    SCMDatanodeHeartbeatDispatcher dispatcher = new SCMDatanodeHeartbeatDispatcher(mockNodeManager, new EventPublisher() {

        @Override
        public <PAYLOAD, EVENT extends Event<PAYLOAD>> void fireEvent(EVENT event, PAYLOAD payload) {
            Assertions.assertEquals(event, NODE_REPORT);
            eventReceived.incrementAndGet();
            Assertions.assertEquals(nodeReport, ((NodeReportFromDatanode) payload).getReport());
        }
    });
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    SCMHeartbeatRequestProto heartbeat = SCMHeartbeatRequestProto.newBuilder().setDatanodeDetails(datanodeDetails.getProtoBufMessage()).setNodeReport(nodeReport).build();
    dispatcher.dispatch(heartbeat);
    Assertions.assertEquals(1, eventReceived.get());
}