Examples with SequenceIdGenerator org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator used on opensource projects

Example 1 with SequenceIdGenerator

use of org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator in project ozone by apache.

the class TestBlockManager method setUp.

@Before
public void setUp() throws Exception {
    conf = SCMTestUtils.getConf();
    numContainerPerOwnerInPipeline = conf.getInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT);
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().toString());
    conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    conf.setTimeDuration(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, 5, TimeUnit.SECONDS);
    // Override the default Node Manager and SCMHAManager
    // in SCM with the Mock one.
    nodeManager = new MockNodeManager(true, 10);
    scmHAManager = MockSCMHAManager.getInstance(true);
    eventQueue = new EventQueue();
    scmContext = SCMContext.emptyContext();
    serviceManager = new SCMServiceManager();
    scmMetadataStore = new SCMMetadataStoreImpl(conf);
    scmMetadataStore.start(conf);
    sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    pipelineManager = PipelineManagerImpl.newPipelineManager(conf, scmHAManager, nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf, eventQueue);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
    ContainerManager containerManager = new ContainerManagerImpl(conf, scmHAManager, sequenceIdGen, pipelineManager, scmMetadataStore.getContainerTable());
    SCMSafeModeManager safeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), containerManager, pipelineManager, eventQueue, serviceManager, scmContext) {

        @Override
        public void emitSafeModeStatus() {
        // skip
        }
    };
    SCMConfigurator configurator = new SCMConfigurator();
    configurator.setScmNodeManager(nodeManager);
    configurator.setPipelineManager(pipelineManager);
    configurator.setContainerManager(containerManager);
    configurator.setScmSafeModeManager(safeModeManager);
    configurator.setMetadataStore(scmMetadataStore);
    configurator.setSCMHAManager(scmHAManager);
    configurator.setScmContext(scmContext);
    scm = HddsTestUtils.getScm(conf, configurator);
    // Initialize these fields so that the tests can pass.
    mapping = scm.getContainerManager();
    blockManager = (BlockManagerImpl) scm.getScmBlockManager();
    DatanodeCommandHandler handler = new DatanodeCommandHandler();
    eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, handler);
    CloseContainerEventHandler closeContainerHandler = new CloseContainerEventHandler(pipelineManager, mapping, scmContext);
    eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
    replicationConfig = RatisReplicationConfig.getInstance(ReplicationFactor.THREE);
    scm.getScmContext().updateSafeModeStatus(new SafeModeStatus(false, true));
}

Example 2 with SequenceIdGenerator

use of org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator in project ozone by apache.

the class TestContainerManagerImpl method setUp.

@Before
public void setUp() throws Exception {
    final OzoneConfiguration conf = SCMTestUtils.getConf();
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    scmhaManager = MockSCMHAManager.getInstance(true);
    nodeManager = new MockNodeManager(true, 10);
    sequenceIdGen = new SequenceIdGenerator(conf, scmhaManager, SCMDBDefinition.SEQUENCE_ID.getTable(dbStore));
    final PipelineManager pipelineManager = new MockPipelineManager(dbStore, scmhaManager, nodeManager);
    pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
    containerManager = new ContainerManagerImpl(conf, scmhaManager, sequenceIdGen, pipelineManager, SCMDBDefinition.CONTAINERS.getTable(dbStore));
}

Example 3 with SequenceIdGenerator

use of org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator in project ozone by apache.

the class TestContainerPlacement method setUp.

@Before
public void setUp() throws Exception {
    conf = getConf();
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    scmhaManager = MockSCMHAManager.getInstance(true);
    sequenceIdGen = new SequenceIdGenerator(conf, scmhaManager, SCMDBDefinition.SEQUENCE_ID.getTable(dbStore));
    nodeManager = new MockNodeManager(true, 10);
    pipelineManager = new MockPipelineManager(dbStore, scmhaManager, nodeManager);
    pipelineManager.createPipeline(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
}

Example 4 with SequenceIdGenerator

use of org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator in project ozone by apache.

the class StorageContainerManager method initializeSystemManagers.

/**
 * This function initializes the following managers. If the configurator
 * specifies a value, we will use it, else we will use the default value.
 *
 *  Node Manager
 *  Pipeline Manager
 *  Container Manager
 *  Block Manager
 *  Replication Manager
 *  Safe Mode Manager
 *
 * @param conf - Ozone Configuration.
 * @param configurator - A customizer which allows different managers to be
 *                    used if needed.
 * @throws IOException - on Failure.
 */
private void initializeSystemManagers(OzoneConfiguration conf, SCMConfigurator configurator) throws IOException {
    if (configurator.getNetworkTopology() != null) {
        clusterMap = configurator.getNetworkTopology();
    } else {
        clusterMap = new NetworkTopologyImpl(conf);
    }
    // This needs to be done before initializing Ratis.
    RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap);
    if (configurator.getSCMHAManager() != null) {
        scmHAManager = configurator.getSCMHAManager();
    } else {
        scmHAManager = new SCMHAManagerImpl(conf, this);
    }
    // inline upgrade for SequenceIdGenerator
    SequenceIdGenerator.upgradeToSequenceId(scmMetadataStore);
    // Distributed sequence id generator
    sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    if (configurator.getScmContext() != null) {
        scmContext = configurator.getScmContext();
    } else {
        // When term equals SCMContext.INVALID_TERM, the isLeader() check
        // and getTermOfLeader() will always pass.
        long term = SCMHAUtils.isSCMHAEnabled(conf) ? 0 : SCMContext.INVALID_TERM;
        // non-leader of term 0, in safe mode, preCheck not completed.
        scmContext = new SCMContext.Builder().setLeader(false).setTerm(term).setIsInSafeMode(true).setIsPreCheckComplete(false).setSCM(this).build();
    }
    if (configurator.getScmNodeManager() != null) {
        scmNodeManager = configurator.getScmNodeManager();
    } else {
        scmNodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, scmContext, scmLayoutVersionManager);
    }
    placementMetrics = SCMContainerPlacementMetrics.create();
    containerPlacementPolicy = ContainerPlacementPolicyFactory.getPolicy(conf, scmNodeManager, clusterMap, true, placementMetrics);
    if (configurator.getPipelineManager() != null) {
        pipelineManager = configurator.getPipelineManager();
    } else {
        pipelineManager = PipelineManagerImpl.newPipelineManager(conf, scmHAManager, scmNodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    }
    if (configurator.getContainerManager() != null) {
        containerManager = configurator.getContainerManager();
    } else {
        containerManager = new ContainerManagerImpl(conf, scmHAManager, sequenceIdGen, pipelineManager, scmMetadataStore.getContainerTable());
    }
    pipelineChoosePolicy = PipelineChoosePolicyFactory.getPolicy(conf);
    if (configurator.getWritableContainerFactory() != null) {
        writableContainerFactory = configurator.getWritableContainerFactory();
    } else {
        writableContainerFactory = new WritableContainerFactory(this);
    }
    if (configurator.getScmBlockManager() != null) {
        scmBlockManager = configurator.getScmBlockManager();
    } else {
        scmBlockManager = new BlockManagerImpl(conf, this);
    }
    if (configurator.getReplicationManager() != null) {
        replicationManager = configurator.getReplicationManager();
    } else {
        replicationManager = new ReplicationManager(conf, containerManager, containerPlacementPolicy, eventQueue, scmContext, serviceManager, scmNodeManager, new MonotonicClock(ZoneOffset.UTC), scmHAManager, getScmMetadataStore().getMoveTable());
    }
    if (configurator.getScmSafeModeManager() != null) {
        scmSafeModeManager = configurator.getScmSafeModeManager();
    } else {
        scmSafeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), containerManager, pipelineManager, eventQueue, serviceManager, scmContext);
    }
    scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, containerManager, scmContext, eventQueue, replicationManager);
}

Example 5 with SequenceIdGenerator

use of org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator in project ozone by apache.

the class TestCloseContainerEventHandler method setUp.

@BeforeClass
public static void setUp() throws Exception {
    configuration = SCMTestUtils.getConf();
    size = (long) configuration.getStorageSize(OZONE_SCM_CONTAINER_SIZE, OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
    testDir = GenericTestUtils.getTestDir(TestCloseContainerEventHandler.class.getSimpleName());
    configuration.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16);
    nodeManager = new MockNodeManager(true, 10);
    eventQueue = new EventQueue();
    scmContext = SCMContext.emptyContext();
    scmMetadataStore = new SCMMetadataStoreImpl(configuration);
    scmhaManager = MockSCMHAManager.getInstance(true);
    sequenceIdGen = new SequenceIdGenerator(configuration, scmhaManager, scmMetadataStore.getSequenceIdTable());
    SCMServiceManager serviceManager = new SCMServiceManager();
    pipelineManager = PipelineManagerImpl.newPipelineManager(configuration, scmhaManager, nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), configuration, eventQueue);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
    containerManager = new ContainerManagerImpl(configuration, scmhaManager, sequenceIdGen, pipelineManager, scmMetadataStore.getContainerTable());
    // trigger BackgroundPipelineCreator to take effect.
    serviceManager.notifyEventTriggered(Event.PRE_CHECK_COMPLETED);
    eventQueue.addHandler(CLOSE_CONTAINER, new CloseContainerEventHandler(pipelineManager, containerManager, scmContext));
    eventQueue.addHandler(DATANODE_COMMAND, nodeManager);
    // Move all pipelines created by background from ALLOCATED to OPEN state
    Thread.sleep(2000);
    HddsTestUtils.openAllRatisPipelines(pipelineManager);
}