Search in sources :

Example 1 with OzoneStorageContainerManager

use of org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager in project ozone by apache.

the class TestReconAsPassiveScm method testReconRestart.

@Test
public void testReconRestart() throws Exception {
    final OzoneStorageContainerManager reconScm = cluster.getReconServer().getReconStorageContainerManager();
    StorageContainerManager scm = cluster.getStorageContainerManager();
    // Stop Recon
    ContainerManager scmContainerManager = scm.getContainerManager();
    assertTrue(scmContainerManager.getContainers().isEmpty());
    ContainerManager reconContainerManager = reconScm.getContainerManager();
    assertTrue(reconContainerManager.getContainers().isEmpty());
    LambdaTestUtils.await(60000, 5000, () -> (reconScm.getScmNodeManager().getAllNodes().size() == 3));
    cluster.stopRecon();
    // Create container in SCM.
    ContainerInfo containerInfo = scmContainerManager.allocateContainer(RatisReplicationConfig.getInstance(ONE), "test");
    long containerID = containerInfo.getContainerID();
    PipelineManager scmPipelineManager = scm.getPipelineManager();
    Pipeline pipeline = scmPipelineManager.getPipeline(containerInfo.getPipelineID());
    XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
    runTestOzoneContainerViaDataNode(containerID, client);
    assertFalse(scmContainerManager.getContainers().isEmpty());
    // Close a pipeline
    Optional<Pipeline> pipelineToClose = scmPipelineManager.getPipelines(RatisReplicationConfig.getInstance(ONE)).stream().filter(p -> !p.getId().equals(containerInfo.getPipelineID())).findFirst();
    assertTrue(pipelineToClose.isPresent());
    scmPipelineManager.closePipeline(pipelineToClose.get(), false);
    // Start Recon
    cluster.startRecon();
    // Verify if Recon has all the nodes on restart (even if heartbeats are
    // not yet received).
    NodeManager reconNodeManager = reconScm.getScmNodeManager();
    NodeManager scmNodeManager = scm.getScmNodeManager();
    assertEquals(scmNodeManager.getAllNodes().size(), reconNodeManager.getAllNodes().size());
    // Verify Recon picks up new container, close pipeline SCM actions.
    OzoneStorageContainerManager newReconScm = cluster.getReconServer().getReconStorageContainerManager();
    PipelineManager reconPipelineManager = newReconScm.getPipelineManager();
    assertFalse(reconPipelineManager.containsPipeline(pipelineToClose.get().getId()));
    LambdaTestUtils.await(90000, 5000, () -> (newReconScm.getContainerManager().containerExist(ContainerID.valueOf(containerID))));
}
Also used : OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) ReconNodeManager(org.apache.hadoop.ozone.recon.scm.ReconNodeManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) MiniOzoneCluster(org.apache.hadoop.ozone.MiniOzoneCluster) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ONE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE) PipelineNotFoundException(org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException) After(org.junit.After) Level(org.slf4j.event.Level) Timeout(org.junit.rules.Timeout) ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) LambdaTestUtils(org.apache.ozone.test.LambdaTestUtils) TestOzoneContainer.runTestOzoneContainerViaDataNode(org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer.runTestOzoneContainerViaDataNode) Before(org.junit.Before) HDDS_CONTAINER_REPORT_INTERVAL(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL) RatisReplicationConfig(org.apache.hadoop.hdds.client.RatisReplicationConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) CLOSE_CONTAINER(org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER) Rule(org.junit.Rule) HDDS_PIPELINE_REPORT_INTERVAL(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) Assert(org.junit.Assert) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) ReconNodeManager(org.apache.hadoop.ozone.recon.scm.ReconNodeManager) NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ContainerManager(org.apache.hadoop.hdds.scm.container.ContainerManager) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test)

Example 2 with OzoneStorageContainerManager

use of org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager in project ozone by apache.

the class TestReconWithOzoneManagerFSO method testNamespaceSummaryAPI.

@Test
public void testNamespaceSummaryAPI() throws Exception {
    // add a vol, bucket and key
    addKeys(0, 10, "dir");
    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    impl.syncDataFromOM();
    ReconNamespaceSummaryManager namespaceSummaryManager = cluster.getReconServer().getReconNamespaceSummaryManager();
    ReconOMMetadataManager omMetadataManagerInstance = (ReconOMMetadataManager) cluster.getReconServer().getOzoneManagerServiceProvider().getOMMetadataManagerInstance();
    OzoneStorageContainerManager reconSCM = cluster.getReconServer().getReconStorageContainerManager();
    NSSummaryEndpoint endpoint = new NSSummaryEndpoint(namespaceSummaryManager, omMetadataManagerInstance, reconSCM);
    Response basicInfo = endpoint.getBasicInfo("/vol1/bucket1/dir1");
    NamespaceSummaryResponse entity = (NamespaceSummaryResponse) basicInfo.getEntity();
    Assert.assertSame(entity.getEntityType(), EntityType.DIRECTORY);
    Assert.assertEquals(1, entity.getNumTotalKey());
    Assert.assertEquals(0, entity.getNumTotalDir());
    for (int i = 0; i < 10; i++) {
        Assert.assertNotNull(impl.getOMMetadataManagerInstance().getVolumeTable().get("/vol" + i));
    }
    addKeys(10, 12, "dir");
    impl.syncDataFromOM();
    // test Recon is sync'ed with OM.
    for (int i = 10; i < 12; i++) {
        Assert.assertNotNull(impl.getOMMetadataManagerInstance().getVolumeTable().getSkipCache("/vol" + i));
    }
    // test root response
    Response rootBasicRes = endpoint.getBasicInfo("/");
    NamespaceSummaryResponse rootBasicEntity = (NamespaceSummaryResponse) rootBasicRes.getEntity();
    Assert.assertSame(EntityType.ROOT, rootBasicEntity.getEntityType());
    // one additional dummy volume at creation
    Assert.assertEquals(13, rootBasicEntity.getNumVolume());
    Assert.assertEquals(12, rootBasicEntity.getNumBucket());
    Assert.assertEquals(12, rootBasicEntity.getNumTotalDir());
    Assert.assertEquals(12, rootBasicEntity.getNumTotalKey());
}
Also used : NamespaceSummaryResponse(org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse) Response(javax.ws.rs.core.Response) NSSummaryEndpoint(org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint) ReconNamespaceSummaryManager(org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager) OzoneManagerServiceProviderImpl(org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl) ReconOMMetadataManager(org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) NSSummaryEndpoint(org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint) NamespaceSummaryResponse(org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse) Test(org.junit.Test)

Example 3 with OzoneStorageContainerManager

use of org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager in project ozone by apache.

the class TestContainerEndpoint method initializeInjector.

private void initializeInjector() throws Exception {
    reconOMMetadataManager = getTestReconOmMetadataManager(initializeNewOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(temporaryFolder).withReconSqlDb().withReconOm(reconOMMetadataManager).withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)).addBinding(OzoneStorageContainerManager.class, ReconStorageContainerManagerFacade.class).withContainerDB().addBinding(StorageContainerServiceProvider.class, mock(StorageContainerServiceProviderImpl.class)).addBinding(ContainerEndpoint.class).addBinding(ContainerHealthSchemaManager.class).build();
    ozoneStorageContainerManager = reconTestInjector.getInstance(OzoneStorageContainerManager.class);
    reconContainerManager = (ReconContainerManager) ozoneStorageContainerManager.getContainerManager();
    reconPipelineManager = (ReconPipelineManager) ozoneStorageContainerManager.getPipelineManager();
    reconContainerMetadataManager = reconTestInjector.getInstance(ReconContainerMetadataManager.class);
    containerEndpoint = reconTestInjector.getInstance(ContainerEndpoint.class);
    containerHealthSchemaManager = reconTestInjector.getInstance(ContainerHealthSchemaManager.class);
    pipeline = getRandomPipeline();
    pipelineID = pipeline.getId();
    reconPipelineManager.addPipeline(pipeline);
}
Also used : ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) ContainerHealthSchemaManager(org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager) ReconContainerMetadataManager(org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager) StorageContainerServiceProvider(org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider) ReconTestInjector(org.apache.hadoop.ozone.recon.ReconTestInjector) OzoneManagerServiceProviderImpl(org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager)

Aggregations

OzoneStorageContainerManager (org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager)3 ReconStorageContainerManagerFacade (org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade)2 OzoneManagerServiceProviderImpl (org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl)2 Test (org.junit.Test)2 Optional (java.util.Optional)1 Response (javax.ws.rs.core.Response)1 HDDS_CONTAINER_REPORT_INTERVAL (org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL)1 HDDS_PIPELINE_REPORT_INTERVAL (org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL)1 RatisReplicationConfig (org.apache.hadoop.hdds.client.RatisReplicationConfig)1 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)1 ONE (org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE)1 XceiverClientGrpc (org.apache.hadoop.hdds.scm.XceiverClientGrpc)1 ContainerID (org.apache.hadoop.hdds.scm.container.ContainerID)1 ContainerInfo (org.apache.hadoop.hdds.scm.container.ContainerInfo)1 ContainerManager (org.apache.hadoop.hdds.scm.container.ContainerManager)1 CLOSE_CONTAINER (org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER)1 NodeManager (org.apache.hadoop.hdds.scm.node.NodeManager)1 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)1 PipelineManager (org.apache.hadoop.hdds.scm.pipeline.PipelineManager)1 PipelineNotFoundException (org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException)1