Examples with SCMMetadataStore org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore used on opensource projects

Example 1 with SCMMetadataStore

use of org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore in project ozone by apache.

the class TestSCMInstallSnapshotWithHA method testInstallSnapshot.

@Test
public void testInstallSnapshot() throws Exception {
    // Get the leader SCM
    StorageContainerManager leaderSCM = getLeader(cluster);
    Assert.assertNotNull(leaderSCM);
    // Find the inactive SCM
    String followerId = getInactiveSCM(cluster).getSCMNodeId();
    StorageContainerManager followerSCM = cluster.getSCM(followerId);
    // Do some transactions so that the log index increases
    List<ContainerInfo> containers = writeToIncreaseLogIndex(leaderSCM, 200);
    // Start the inactive SCM. Install Snapshot will happen as part
    // of setConfiguration() call to ratis leader and the follower will catch
    // up
    cluster.startInactiveSCM(followerId);
    // The recently started  should be lagging behind the leader .
    SCMStateMachine followerSM = followerSCM.getScmHAManager().getRatisServer().getSCMStateMachine();
    // Wait & retry for follower to update transactions to leader
    // snapshot index.
    // Timeout error if follower does not load update within 3s
    GenericTestUtils.waitFor(() -> {
        return followerSM.getLastAppliedTermIndex().getIndex() >= 200;
    }, 100, 3000);
    long followerLastAppliedIndex = followerSM.getLastAppliedTermIndex().getIndex();
    assertTrue(followerLastAppliedIndex >= 200);
    assertFalse(followerSM.getLifeCycleState().isPausingOrPaused());
    // Verify that the follower 's DB contains the transactions which were
    // made while it was inactive.
    SCMMetadataStore followerMetaStore = followerSCM.getScmMetadataStore();
    for (ContainerInfo containerInfo : containers) {
        Assert.assertNotNull(followerMetaStore.getContainerTable().get(containerInfo.containerID()));
    }
}

Example 2 with SCMMetadataStore

use of org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore in project ozone by apache.

the class TestHealthyPipelineSafeModeRule method testHealthyPipelineSafeModeRuleWithMixedPipelines.

@Test
public void testHealthyPipelineSafeModeRuleWithMixedPipelines() throws Exception {
    String storageDir = GenericTestUtils.getTempPath(TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
    EventQueue eventQueue = new EventQueue();
    SCMServiceManager serviceManager = new SCMServiceManager();
    SCMContext scmContext = SCMContext.emptyContext();
    List<ContainerInfo> containers = new ArrayList<>(HddsTestUtils.getContainerInfo(1));
    OzoneConfiguration config = new OzoneConfiguration();
    // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are
    // stale and last one is dead, and this repeats. So for a 12 node, 9
    // healthy, 2 stale and one dead.
    MockNodeManager nodeManager = new MockNodeManager(true, 12);
    config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
    // enable pipeline check
    config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
    config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config);
    try {
        PipelineManagerImpl pipelineManager = PipelineManagerImpl.newPipelineManager(config, MockSCMHAManager.getInstance(true), nodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
        PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config);
        pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
        // Create 3 pipelines
        Pipeline pipeline1 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.ONE));
        pipelineManager.openPipeline(pipeline1.getId());
        Pipeline pipeline2 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
        pipelineManager.openPipeline(pipeline2.getId());
        Pipeline pipeline3 = pipelineManager.createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
        pipelineManager.openPipeline(pipeline3.getId());
        // Mark pipeline healthy
        pipeline1 = pipelineManager.getPipeline(pipeline1.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline1);
        pipeline2 = pipelineManager.getPipeline(pipeline2.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline2);
        pipeline3 = pipelineManager.getPipeline(pipeline3.getId());
        MockRatisPipelineProvider.markPipelineHealthy(pipeline3);
        SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(config, containers, null, pipelineManager, eventQueue, serviceManager, scmContext);
        HealthyPipelineSafeModeRule healthyPipelineSafeModeRule = scmSafeModeManager.getHealthyPipelineSafeModeRule();
        // No pipeline event have sent to SCMSafemodeManager
        Assert.assertFalse(healthyPipelineSafeModeRule.validate());
        GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(LoggerFactory.getLogger(SCMSafeModeManager.class));
        // fire event with pipeline create status with ratis type and factor 1
        // pipeline, validate() should return false
        firePipelineEvent(pipeline1, eventQueue);
        GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains("reported count is 1"), 1000, 5000);
        Assert.assertFalse(healthyPipelineSafeModeRule.validate());
        firePipelineEvent(pipeline2, eventQueue);
        firePipelineEvent(pipeline3, eventQueue);
        GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000);
    } finally {
        scmMetadataStore.getStore().close();
        FileUtil.fullyDelete(new File(storageDir));
    }
}

Example 3 with SCMMetadataStore

use of org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore in project ozone by apache.

the class TestOneReplicaPipelineSafeModeRule method setup.

private void setup(int nodes, int pipelineFactorThreeCount, int pipelineFactorOneCount) throws Exception {
    OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
    ozoneConfiguration.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
    ozoneConfiguration.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().toString());
    ozoneConfiguration.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
    List<ContainerInfo> containers = new ArrayList<>();
    containers.addAll(HddsTestUtils.getContainerInfo(1));
    mockNodeManager = new MockNodeManager(true, nodes);
    eventQueue = new EventQueue();
    serviceManager = new SCMServiceManager();
    scmContext = SCMContext.emptyContext();
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(ozoneConfiguration);
    pipelineManager = PipelineManagerImpl.newPipelineManager(ozoneConfiguration, MockSCMHAManager.getInstance(true), mockNodeManager, scmMetadataStore.getPipelineTable(), eventQueue, scmContext, serviceManager);
    PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), ozoneConfiguration);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);
    createPipelines(pipelineFactorThreeCount, HddsProtos.ReplicationFactor.THREE);
    createPipelines(pipelineFactorOneCount, HddsProtos.ReplicationFactor.ONE);
    SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(ozoneConfiguration, containers, null, pipelineManager, eventQueue, serviceManager, scmContext);
    rule = scmSafeModeManager.getOneReplicaPipelineSafeModeRule();
}

Example 4 with SCMMetadataStore

use of org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore in project ozone by apache.

the class TestSequenceIDGenerator method testSequenceIDGenUponRatis.

@Test
public void testSequenceIDGenUponRatis() throws Exception {
    OzoneConfiguration conf = SCMTestUtils.getConf();
    // change batchSize to 100
    conf.setInt(OZONE_SCM_SEQUENCE_ID_BATCH_SIZE, 100);
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf);
    scmMetadataStore.start(conf);
    SCMHAManager scmHAManager = MockSCMHAManager.getInstance(true);
    SequenceIdGenerator sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    // the first batch is [1, 100]
    Assert.assertEquals(1L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(2L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(3L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(1L, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(2L, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(3L, sequenceIdGen.getNextId("otherKey"));
    // the next batch is [101, 200]
    sequenceIdGen.invalidateBatch();
    Assert.assertEquals(101, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(102, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(103, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(101, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(102, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(103, sequenceIdGen.getNextId("otherKey"));
    // the next batch is [201, 300]
    sequenceIdGen.invalidateBatch();
    Assert.assertEquals(201, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(202, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(203, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(201, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(202, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(203, sequenceIdGen.getNextId("otherKey"));
}

Example 5 with SCMMetadataStore

use of org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore in project ozone by apache.

the class TestSequenceIDGenerator method testSequenceIDGenUponNonRatis.

@Test
public void testSequenceIDGenUponNonRatis() throws Exception {
    OzoneConfiguration conf = SCMTestUtils.getConf();
    SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf);
    scmMetadataStore.start(conf);
    SCMHAManager scmHAManager = MockSCMHAManager.getInstance(true, new SCMDBTransactionBufferImpl());
    SequenceIdGenerator sequenceIdGen = new SequenceIdGenerator(conf, scmHAManager, scmMetadataStore.getSequenceIdTable());
    // the first batch is [1, 1000]
    Assert.assertEquals(1L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(2L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(3L, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(1L, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(2L, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(3L, sequenceIdGen.getNextId("otherKey"));
    // default batchSize is 1000, the next batch is [1001, 2000]
    sequenceIdGen.invalidateBatch();
    Assert.assertEquals(1001, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(1002, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(1003, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(1001, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(1002, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(1003, sequenceIdGen.getNextId("otherKey"));
    // default batchSize is 1000, the next batch is [2001, 3000]
    sequenceIdGen.invalidateBatch();
    Assert.assertEquals(2001, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(2002, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(2003, sequenceIdGen.getNextId("someKey"));
    Assert.assertEquals(2001, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(2002, sequenceIdGen.getNextId("otherKey"));
    Assert.assertEquals(2003, sequenceIdGen.getNextId("otherKey"));
}