Examples with StorageContainerManager org.apache.hadoop.hdds.scm.server.StorageContainerManager used on opensource projects

Example 36 with StorageContainerManager

use of org.apache.hadoop.hdds.scm.server.StorageContainerManager in project ozone by apache.

the class TestStorageContainerManagerHA method testPrimordialSCM.

@Test
public void testPrimordialSCM() throws Exception {
    StorageContainerManager scm1 = cluster.getStorageContainerManagers().get(0);
    StorageContainerManager scm2 = cluster.getStorageContainerManagers().get(1);
    OzoneConfiguration conf1 = scm1.getConfiguration();
    OzoneConfiguration conf2 = scm2.getConfiguration();
    conf1.set(ScmConfigKeys.OZONE_SCM_PRIMORDIAL_NODE_ID_KEY, scm1.getSCMNodeId());
    conf2.set(ScmConfigKeys.OZONE_SCM_PRIMORDIAL_NODE_ID_KEY, scm1.getSCMNodeId());
    Assert.assertTrue(StorageContainerManager.scmBootstrap(conf1));
    scm1.getScmHAManager().shutdown();
    Assert.assertTrue(StorageContainerManager.scmInit(conf1, scm1.getClusterId()));
    Assert.assertTrue(StorageContainerManager.scmBootstrap(conf2));
    Assert.assertTrue(StorageContainerManager.scmInit(conf2, scm2.getClusterId()));
}

Example 37 with StorageContainerManager

use of org.apache.hadoop.hdds.scm.server.StorageContainerManager in project ozone by apache.

the class TestCloseContainer method testReplicasAreReportedForClosedContainerAfterRestart.

@Test
public void testReplicasAreReportedForClosedContainerAfterRestart() throws Exception {
    // Create some keys to write data into the open containers
    for (int i = 0; i < 10; i++) {
        TestDataUtil.createKey(bucket, "key" + i, ReplicationFactor.THREE, ReplicationType.RATIS, "this is the content");
    }
    StorageContainerManager scm = cluster.getStorageContainerManager();
    // Pick any container on the cluster, get its pipeline, close it and then
    // wait for the container to close
    ContainerInfo container = scm.getContainerManager().getContainers().get(0);
    Pipeline pipeline = scm.getPipelineManager().getPipeline(container.getPipelineID());
    scm.getPipelineManager().closePipeline(pipeline, false);
    GenericTestUtils.waitFor(() -> container.getState() == HddsProtos.LifeCycleState.CLOSED, 200, 30000);
    long originalSeq = container.getSequenceId();
    cluster.restartStorageContainerManager(true);
    scm = cluster.getStorageContainerManager();
    ContainerInfo newContainer = scm.getContainerManager().getContainer(container.containerID());
    // After restarting SCM, ensure the sequenceId for the container is the
    // same as before.
    assertEquals(originalSeq, newContainer.getSequenceId());
    // Ensure 3 replicas are reported successfully as expected.
    GenericTestUtils.waitFor(() -> getContainerReplicas(newContainer).size() == 3, 200, 30000);
}

Example 38 with StorageContainerManager

use of org.apache.hadoop.hdds.scm.server.StorageContainerManager in project ozone by apache.

the class TestFailoverWithSCMHA method testMoveFailover.

@Test
public void testMoveFailover() throws Exception {
    SCMClientConfig scmClientConfig = conf.getObject(SCMClientConfig.class);
    scmClientConfig.setRetryCount(1);
    scmClientConfig.setRetryInterval(100);
    scmClientConfig.setMaxRetryTimeout(1500);
    Assert.assertEquals(scmClientConfig.getRetryCount(), 15);
    conf.setFromObject(scmClientConfig);
    StorageContainerManager scm = getLeader(cluster);
    Assert.assertNotNull(scm);
    final ContainerID id = getContainer(HddsProtos.LifeCycleState.CLOSED).containerID();
    DatanodeDetails dn1 = randomDatanodeDetails();
    DatanodeDetails dn2 = randomDatanodeDetails();
    // here we just want to test whether the new leader will get the same
    // inflight move after failover, so no need to create container and datanode,
    // just mock them bypassing all the pre checks.
    scm.getReplicationManager().getMoveScheduler().startMove(id.getProtobuf(), (new MoveDataNodePair(dn1, dn2)).getProtobufMessage(CURRENT_VERSION));
    SCMBlockLocationFailoverProxyProvider failoverProxyProvider = new SCMBlockLocationFailoverProxyProvider(conf);
    failoverProxyProvider.changeCurrentProxy(scm.getSCMNodeId());
    ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient = new ScmBlockLocationProtocolClientSideTranslatorPB(failoverProxyProvider);
    GenericTestUtils.setLogLevel(SCMBlockLocationFailoverProxyProvider.LOG, Level.DEBUG);
    GenericTestUtils.LogCapturer logCapture = GenericTestUtils.LogCapturer.captureLogs(SCMBlockLocationFailoverProxyProvider.LOG);
    ScmBlockLocationProtocol scmBlockLocationProtocol = TracingUtil.createProxy(scmBlockLocationClient, ScmBlockLocationProtocol.class, conf);
    scmBlockLocationProtocol.getScmInfo();
    Assert.assertTrue(logCapture.getOutput().contains("Performing failover to suggested leader"));
    scm = getLeader(cluster);
    Assert.assertNotNull(scm);
    // switch to the new leader successfully, new leader should
    // get the same inflightMove
    Map<ContainerID, MoveDataNodePair> inflightMove = scm.getReplicationManager().getMoveScheduler().getInflightMove();
    Assert.assertTrue(inflightMove.containsKey(id));
    MoveDataNodePair mp = inflightMove.get(id);
    Assert.assertTrue(dn2.equals(mp.getTgt()));
    Assert.assertTrue(dn1.equals(mp.getSrc()));
    // complete move in the new leader
    scm.getReplicationManager().getMoveScheduler().completeMove(id.getProtobuf());
    SCMContainerLocationFailoverProxyProvider proxyProvider = new SCMContainerLocationFailoverProxyProvider(conf, null);
    GenericTestUtils.setLogLevel(SCMContainerLocationFailoverProxyProvider.LOG, Level.DEBUG);
    logCapture = GenericTestUtils.LogCapturer.captureLogs(SCMContainerLocationFailoverProxyProvider.LOG);
    proxyProvider.changeCurrentProxy(scm.getSCMNodeId());
    StorageContainerLocationProtocol scmContainerClient = TracingUtil.createProxy(new StorageContainerLocationProtocolClientSideTranslatorPB(proxyProvider), StorageContainerLocationProtocol.class, conf);
    scmContainerClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE, "ozone");
    Assert.assertTrue(logCapture.getOutput().contains("Performing failover to suggested leader"));
    // switch to the new leader successfully, new leader should
    // get the same inflightMove , which should not contains
    // that container.
    scm = getLeader(cluster);
    Assert.assertNotNull(scm);
    inflightMove = scm.getReplicationManager().getMoveScheduler().getInflightMove();
    Assert.assertFalse(inflightMove.containsKey(id));
}

Example 39 with StorageContainerManager

use of org.apache.hadoop.hdds.scm.server.StorageContainerManager in project ozone by apache.

the class TestSCMMXBean method testSCMContainerStateCount.

@Test
public void testSCMContainerStateCount() throws Exception {
    ObjectName bean = new ObjectName("Hadoop:service=StorageContainerManager," + "name=StorageContainerManagerInfo," + "component=ServerRuntime");
    TabularData data = (TabularData) mbs.getAttribute(bean, "ContainerStateCount");
    Map<String, Integer> containerStateCount = scm.getContainerStateCount();
    verifyEquals(data, containerStateCount);
    // Do some changes like allocate containers and change the container states
    ContainerManager scmContainerManager = scm.getContainerManager();
    List<ContainerInfo> containerInfoList = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        containerInfoList.add(scmContainerManager.allocateContainer(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE), UUID.randomUUID().toString()));
    }
    long containerID;
    for (int i = 0; i < 10; i++) {
        if (i % 2 == 0) {
            containerID = containerInfoList.get(i).getContainerID();
            scmContainerManager.updateContainerState(ContainerID.valueOf(containerID), HddsProtos.LifeCycleEvent.FINALIZE);
            assertEquals(scmContainerManager.getContainer(ContainerID.valueOf(containerID)).getState(), HddsProtos.LifeCycleState.CLOSING);
        } else {
            containerID = containerInfoList.get(i).getContainerID();
            scmContainerManager.updateContainerState(ContainerID.valueOf(containerID), HddsProtos.LifeCycleEvent.FINALIZE);
            scmContainerManager.updateContainerState(ContainerID.valueOf(containerID), HddsProtos.LifeCycleEvent.CLOSE);
            assertEquals(scmContainerManager.getContainer(ContainerID.valueOf(containerID)).getState(), HddsProtos.LifeCycleState.CLOSED);
        }
    }
    data = (TabularData) mbs.getAttribute(bean, "ContainerStateCount");
    containerStateCount = scm.getContainerStateCount();
    containerStateCount.forEach((k, v) -> {
        if (k.equals(HddsProtos.LifeCycleState.CLOSING.toString())) {
            assertEquals((int) v, 5);
        } else if (k.equals(HddsProtos.LifeCycleState.CLOSED.toString())) {
            assertEquals((int) v, 5);
        } else {
            // Remaining all container state count should be zero.
            assertEquals((int) v, 0);
        }
    });
    verifyEquals(data, containerStateCount);
}