Search in sources :

Example 1 with ContainerManagerImpl

use of org.apache.hadoop.hdds.scm.container.ContainerManagerImpl in project ozone by apache.

the class TestBlockManager method setUp.

@Before
public void setUp() throws Exception {
    conf = SCMTestUtils.getConf();
    numContainerPerOwnerInPipeline = conf.getInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT);
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().toString());
    conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    conf.setTimeDuration(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, 5, TimeUnit.SECONDS);
    // Override the default Node Manager and SCMHAManager
    // in SCM with the Mock one.
    nodeManager = new MockNodeManager(true, 10);
    scmHAManager = MockSCMHAManager.getInstance(true);
    eventQueue = new EventQueue();
    scmContext = SCMContext.emptyContext();
    serviceManager = new SCMServiceManager();
    scmMetadataStore = new SCMMetadataStoreImpl(conf);
    scmMetadataStore.start(conf);
    sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    pipelineManager = PipelineManagerImpl.newPipelineManager(conf, scmHAManager, nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf, eventQueue);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
    ContainerManager containerManager = new ContainerManagerImpl(conf, scmHAManager, sequenceIdGen, pipelineManager, scmMetadataStore.getContainerTable());
    SCMSafeModeManager safeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), containerManager, pipelineManager, eventQueue, serviceManager, scmContext) {

        @Override
        public void emitSafeModeStatus() {
        // skip
        }
    };
    SCMConfigurator configurator = new SCMConfigurator();
    configurator.setScmNodeManager(nodeManager);
    configurator.setPipelineManager(pipelineManager);
    configurator.setContainerManager(containerManager);
    configurator.setScmSafeModeManager(safeModeManager);
    configurator.setMetadataStore(scmMetadataStore);
    configurator.setSCMHAManager(scmHAManager);
    configurator.setScmContext(scmContext);
    scm = HddsTestUtils.getScm(conf, configurator);
    // Initialize these fields so that the tests can pass.
    mapping = scm.getContainerManager();
    blockManager = (BlockManagerImpl) scm.getScmBlockManager();
    DatanodeCommandHandler handler = new DatanodeCommandHandler();
    eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, handler);
    CloseContainerEventHandler closeContainerHandler = new CloseContainerEventHandler(pipelineManager, mapping, scmContext);
    eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
    replicationConfig = RatisReplicationConfig.getInstance(ReplicationFactor.THREE);
    scm.getScmContext().updateSafeModeStatus(new SafeModeStatus(false, true));
}
Also used : ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) PipelineProvider(org.apache.hadoop.hdds.scm.pipeline.PipelineProvider) MockNodeManager(org.apache.hadoop.hdds.scm.container.MockNodeManager) EventQueue(org.apache.hadoop.hdds.server.events.EventQueue) SCMSafeModeManager(org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager) ContainerManagerImpl(org.apache.hadoop.hdds.scm.container.ContainerManagerImpl) CloseContainerEventHandler(org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler) SCMMetadataStoreImpl(org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl) MockRatisPipelineProvider(org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider) SequenceIdGenerator(org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator) SafeModeStatus(org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) SCMConfigurator(org.apache.hadoop.hdds.scm.server.SCMConfigurator) Before(org.junit.Before)

Example 2 with ContainerManagerImpl

use of org.apache.hadoop.hdds.scm.container.ContainerManagerImpl in project ozone by apache.

the class StorageContainerManager method initializeSystemManagers.

/**
 * This function initializes the following managers. If the configurator
 * specifies a value, we will use it, else we will use the default value.
 *
 *  Node Manager
 *  Pipeline Manager
 *  Container Manager
 *  Block Manager
 *  Replication Manager
 *  Safe Mode Manager
 *
 * @param conf - Ozone Configuration.
 * @param configurator - A customizer which allows different managers to be
 *                    used if needed.
 * @throws IOException - on Failure.
 */
private void initializeSystemManagers(OzoneConfiguration conf, SCMConfigurator configurator) throws IOException {
    if (configurator.getNetworkTopology() != null) {
        clusterMap = configurator.getNetworkTopology();
    } else {
        clusterMap = new NetworkTopologyImpl(conf);
    }
    // This needs to be done before initializing Ratis.
    RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap);
    if (configurator.getSCMHAManager() != null) {
        scmHAManager = configurator.getSCMHAManager();
    } else {
        scmHAManager = new SCMHAManagerImpl(conf, this);
    }
    // inline upgrade for SequenceIdGenerator
    SequenceIdGenerator.upgradeToSequenceId(scmMetadataStore);
    // Distributed sequence id generator
    sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    if (configurator.getScmContext() != null) {
        scmContext = configurator.getScmContext();
    } else {
        // When term equals SCMContext.INVALID_TERM, the isLeader() check
        // and getTermOfLeader() will always pass.
        long term = SCMHAUtils.isSCMHAEnabled(conf) ? 0 : SCMContext.INVALID_TERM;
        // non-leader of term 0, in safe mode, preCheck not completed.
        scmContext = new SCMContext.Builder().setLeader(false).setTerm(term).setIsInSafeMode(true).setIsPreCheckComplete(false).setSCM(this).build();
    }
    if (configurator.getScmNodeManager() != null) {
        scmNodeManager = configurator.getScmNodeManager();
    } else {
        scmNodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, scmContext, scmLayoutVersionManager);
    }
    placementMetrics = SCMContainerPlacementMetrics.create();
    containerPlacementPolicy = ContainerPlacementPolicyFactory.getPolicy(conf, scmNodeManager, clusterMap, true, placementMetrics);
    if (configurator.getPipelineManager() != null) {
        pipelineManager = configurator.getPipelineManager();
    } else {
        pipelineManager = PipelineManagerImpl.newPipelineManager(conf, scmHAManager, scmNodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    }
    if (configurator.getContainerManager() != null) {
        containerManager = configurator.getContainerManager();
    } else {
        containerManager = new ContainerManagerImpl(conf, scmHAManager, sequenceIdGen, pipelineManager, scmMetadataStore.getContainerTable());
    }
    pipelineChoosePolicy = PipelineChoosePolicyFactory.getPolicy(conf);
    if (configurator.getWritableContainerFactory() != null) {
        writableContainerFactory = configurator.getWritableContainerFactory();
    } else {
        writableContainerFactory = new WritableContainerFactory(this);
    }
    if (configurator.getScmBlockManager() != null) {
        scmBlockManager = configurator.getScmBlockManager();
    } else {
        scmBlockManager = new BlockManagerImpl(conf, this);
    }
    if (configurator.getReplicationManager() != null) {
        replicationManager = configurator.getReplicationManager();
    } else {
        replicationManager = new ReplicationManager(conf, containerManager, containerPlacementPolicy, eventQueue, scmContext, serviceManager, scmNodeManager, new MonotonicClock(ZoneOffset.UTC), scmHAManager, getScmMetadataStore().getMoveTable());
    }
    if (configurator.getScmSafeModeManager() != null) {
        scmSafeModeManager = configurator.getScmSafeModeManager();
    } else {
        scmSafeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), containerManager, pipelineManager, eventQueue, serviceManager, scmContext);
    }
    scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, containerManager, scmContext, eventQueue, replicationManager);
}
Also used : ReplicationManager(org.apache.hadoop.hdds.scm.container.ReplicationManager) SCMHAManagerImpl(org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl) SCMSafeModeManager(org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager) ContainerManagerImpl(org.apache.hadoop.hdds.scm.container.ContainerManagerImpl) NetworkTopologyImpl(org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl) SCMContext(org.apache.hadoop.hdds.scm.ha.SCMContext) SCMNodeManager(org.apache.hadoop.hdds.scm.node.SCMNodeManager) SequenceIdGenerator(org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator) WritableContainerFactory(org.apache.hadoop.hdds.scm.pipeline.WritableContainerFactory) NodeDecommissionManager(org.apache.hadoop.hdds.scm.node.NodeDecommissionManager) MonotonicClock(org.apache.hadoop.ozone.common.MonotonicClock) BlockManagerImpl(org.apache.hadoop.hdds.scm.block.BlockManagerImpl)

Aggregations

ContainerManagerImpl (org.apache.hadoop.hdds.scm.container.ContainerManagerImpl)2 SequenceIdGenerator (org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator)2 SCMSafeModeManager (org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager)2 BlockManagerImpl (org.apache.hadoop.hdds.scm.block.BlockManagerImpl)1 CloseContainerEventHandler (org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler)1 ContainerManager (org.apache.hadoop.hdds.scm.container.ContainerManager)1 MockNodeManager (org.apache.hadoop.hdds.scm.container.MockNodeManager)1 ReplicationManager (org.apache.hadoop.hdds.scm.container.ReplicationManager)1 SCMContext (org.apache.hadoop.hdds.scm.ha.SCMContext)1 SCMHAManagerImpl (org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl)1 SCMServiceManager (org.apache.hadoop.hdds.scm.ha.SCMServiceManager)1 SCMMetadataStoreImpl (org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl)1 NetworkTopologyImpl (org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl)1 NodeDecommissionManager (org.apache.hadoop.hdds.scm.node.NodeDecommissionManager)1 SCMNodeManager (org.apache.hadoop.hdds.scm.node.SCMNodeManager)1 MockRatisPipelineProvider (org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider)1 PipelineProvider (org.apache.hadoop.hdds.scm.pipeline.PipelineProvider)1 WritableContainerFactory (org.apache.hadoop.hdds.scm.pipeline.WritableContainerFactory)1 SafeModeStatus (org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus)1 SCMConfigurator (org.apache.hadoop.hdds.scm.server.SCMConfigurator)1