Examples with ReconStorageContainerManagerFacade org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade used on opensource projects

Example 1 with ReconStorageContainerManagerFacade

use of org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade in project ozone by apache.

the class TestReconAsPassiveScm method testDatanodeRegistrationAndReports.

@Test
public void testDatanodeRegistrationAndReports() throws Exception {
    ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) cluster.getReconServer().getReconStorageContainerManager();
    StorageContainerManager scm = cluster.getStorageContainerManager();
    PipelineManager reconPipelineManager = reconScm.getPipelineManager();
    PipelineManager scmPipelineManager = scm.getPipelineManager();
    LambdaTestUtils.await(60000, 5000, () -> (reconPipelineManager.getPipelines().size() >= 4));
    // Verify if Recon has all the pipelines from SCM.
    scmPipelineManager.getPipelines().forEach(p -> {
        try {
            assertNotNull(reconPipelineManager.getPipeline(p.getId()));
        } catch (PipelineNotFoundException e) {
            Assert.fail();
        }
    });
    // Verify we can never create a pipeline in Recon.
    LambdaTestUtils.intercept(UnsupportedOperationException.class, "Trying to create pipeline in Recon, which is prohibited!", () -> reconPipelineManager.createPipeline(RatisReplicationConfig.getInstance(ONE)));
    ContainerManager scmContainerManager = scm.getContainerManager();
    assertTrue(scmContainerManager.getContainers().isEmpty());
    // Verify if all the 3 nodes are registered with Recon.
    NodeManager reconNodeManager = reconScm.getScmNodeManager();
    NodeManager scmNodeManager = scm.getScmNodeManager();
    assertEquals(scmNodeManager.getAllNodes().size(), reconNodeManager.getAllNodes().size());
    // Create container
    ContainerManager reconContainerManager = reconScm.getContainerManager();
    ContainerInfo containerInfo = scmContainerManager.allocateContainer(RatisReplicationConfig.getInstance(ONE), "test");
    long containerID = containerInfo.getContainerID();
    Pipeline pipeline = scmPipelineManager.getPipeline(containerInfo.getPipelineID());
    XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
    runTestOzoneContainerViaDataNode(containerID, client);
    // Verify Recon picked up the new container that was created.
    assertEquals(scmContainerManager.getContainerIDs(), reconContainerManager.getContainerIDs());
    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(ReconNodeManager.LOG);
    GenericTestUtils.setLogLevel(ReconNodeManager.LOG, Level.DEBUG);
    reconScm.getEventQueue().fireEvent(CLOSE_CONTAINER, containerInfo.containerID());
    GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains("Ignoring unsupported command closeContainerCommand"), 1000, 20000);
}

Example 2 with ReconStorageContainerManagerFacade

use of org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade in project ozone by apache.

the class TestNSSummaryEndpoint method getMockReconSCM.

private static ReconStorageContainerManagerFacade getMockReconSCM() throws ContainerNotFoundException {
    ReconStorageContainerManagerFacade reconSCM = mock(ReconStorageContainerManagerFacade.class);
    ContainerManager containerManager = mock(ContainerManager.class);
    // Container 1 is 3-way replicated
    ContainerID containerID1 = new ContainerID(CONTAINER_ONE_ID);
    Set<ContainerReplica> containerReplicas1 = generateMockContainerReplicas(THREE, containerID1);
    when(containerManager.getContainerReplicas(containerID1)).thenReturn(containerReplicas1);
    // Container 2 is under replicated with 2 replica
    ContainerID containerID2 = new ContainerID(CONTAINER_TWO_ID);
    Set<ContainerReplica> containerReplicas2 = generateMockContainerReplicas(TWO, containerID2);
    when(containerManager.getContainerReplicas(containerID2)).thenReturn(containerReplicas2);
    // Container 3 is over replicated with 4 replica
    ContainerID containerID3 = new ContainerID(CONTAINER_THREE_ID);
    Set<ContainerReplica> containerReplicas3 = generateMockContainerReplicas(FOUR, containerID3);
    when(containerManager.getContainerReplicas(containerID3)).thenReturn(containerReplicas3);
    when(reconSCM.getContainerManager()).thenReturn(containerManager);
    return reconSCM;
}

Example 3 with ReconStorageContainerManagerFacade

use of org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade in project ozone by apache.

the class TestOpenContainerCount method initializeInjector.

private void initializeInjector() throws Exception {
    reconOMMetadataManager = getTestReconOmMetadataManager(initializeNewOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    datanodeDetails = randomDatanodeDetails();
    datanodeDetails.setHostName(HOST1);
    datanodeDetails.setIpAddress(IP1);
    pipeline = getRandomPipeline(datanodeDetails);
    pipelineId = pipeline.getId().getId().toString();
    pipeline2 = getRandomPipeline(datanodeDetails);
    pipelineId2 = pipeline2.getId().getId().toString();
    StorageContainerLocationProtocol mockScmClient = mock(StorageContainerLocationProtocol.class);
    mockScmServiceProvider = mock(StorageContainerServiceProviderImpl.class);
    when(mockScmServiceProvider.getPipeline(pipeline.getId().getProtobuf())).thenReturn(pipeline);
    when(mockScmServiceProvider.getPipeline(pipeline2.getId().getProtobuf())).thenReturn(pipeline2);
    // Open 5 containers on pipeline 1
    containerIDs = new LinkedList<>();
    cpw = new LinkedList<>();
    for (long i = 1L; i <= 5L; ++i) {
        ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(i).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline.getId()).build();
        ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline);
        when(mockScmServiceProvider.getContainerWithPipeline(i)).thenReturn(containerWithPipeline);
        containerIDs.add(i);
        cpw.add(containerWithPipeline);
    }
    // Open 5 containers on pipeline 2
    for (long i = 6L; i <= 10L; ++i) {
        ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(i).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline2.getId()).build();
        ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline2);
        when(mockScmServiceProvider.getContainerWithPipeline(i)).thenReturn(containerWithPipeline);
        containerIDs.add(i);
        cpw.add(containerWithPipeline);
    }
    when(mockScmServiceProvider.getExistContainerWithPipelinesInBatch(containerIDs)).thenReturn(cpw);
    reconUtilsMock = mock(ReconUtils.class);
    HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class);
    when(urlConnectionMock.getResponseCode()).thenReturn(HttpServletResponse.SC_OK);
    when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), anyString(), anyBoolean())).thenReturn(urlConnectionMock);
    when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), anyString())).thenReturn(GenericTestUtils.getRandomizedTestDir());
    ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(temporaryFolder).withReconSqlDb().withReconOm(reconOMMetadataManager).withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)).addBinding(StorageContainerServiceProvider.class, mockScmServiceProvider).addBinding(OzoneStorageContainerManager.class, ReconStorageContainerManagerFacade.class).withContainerDB().addBinding(NodeEndpoint.class).addBinding(MetricsServiceProviderFactory.class).addBinding(ContainerHealthSchemaManager.class).addBinding(ReconUtils.class, reconUtilsMock).addBinding(StorageContainerLocationProtocol.class, mockScmClient).build();
    nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
    reconScm = (ReconStorageContainerManagerFacade) reconTestInjector.getInstance(OzoneStorageContainerManager.class);
}

Example 4 with ReconStorageContainerManagerFacade

use of org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade in project ozone by apache.

the class TestEndpoints method initializeInjector.

private void initializeInjector() throws Exception {
    reconOMMetadataManager = getTestReconOmMetadataManager(initializeNewOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    datanodeDetails = randomDatanodeDetails();
    datanodeDetails2 = randomDatanodeDetails();
    datanodeDetails.setHostName(HOST1);
    datanodeDetails.setIpAddress(IP1);
    datanodeDetails2.setHostName(HOST2);
    datanodeDetails2.setIpAddress(IP2);
    pipeline = getRandomPipeline(datanodeDetails);
    pipelineId = pipeline.getId().getId().toString();
    ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(containerId).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline.getId()).build();
    ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline);
    StorageContainerLocationProtocol mockScmClient = mock(StorageContainerLocationProtocol.class);
    StorageContainerServiceProvider mockScmServiceProvider = mock(StorageContainerServiceProviderImpl.class);
    when(mockScmServiceProvider.getPipeline(pipeline.getId().getProtobuf())).thenReturn(pipeline);
    when(mockScmServiceProvider.getContainerWithPipeline(containerId)).thenReturn(containerWithPipeline);
    List<Long> containerIDs = new LinkedList<>();
    containerIDs.add(containerId);
    List<ContainerWithPipeline> cpw = new LinkedList<>();
    cpw.add(containerWithPipeline);
    when(mockScmServiceProvider.getExistContainerWithPipelinesInBatch(containerIDs)).thenReturn(cpw);
    InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE);
    reconUtilsMock = mock(ReconUtils.class);
    HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class);
    when(urlConnectionMock.getResponseCode()).thenReturn(HttpServletResponse.SC_OK);
    when(urlConnectionMock.getInputStream()).thenReturn(inputStream);
    when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), anyString(), anyBoolean())).thenReturn(urlConnectionMock);
    when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), anyString())).thenReturn(GenericTestUtils.getRandomizedTestDir());
    ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(temporaryFolder).withReconSqlDb().withReconOm(reconOMMetadataManager).withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)).addBinding(StorageContainerServiceProvider.class, mockScmServiceProvider).addBinding(OzoneStorageContainerManager.class, ReconStorageContainerManagerFacade.class).withContainerDB().addBinding(ClusterStateEndpoint.class).addBinding(NodeEndpoint.class).addBinding(MetricsServiceProviderFactory.class).addBinding(ContainerHealthSchemaManager.class).addBinding(UtilizationEndpoint.class).addBinding(ReconUtils.class, reconUtilsMock).addBinding(StorageContainerLocationProtocol.class, mockScmClient).build();
    nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
    pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class);
    fileCountBySizeDao = getDao(FileCountBySizeDao.class);
    GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class);
    UtilizationSchemaDefinition utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class);
    Configuration sqlConfiguration = reconTestInjector.getInstance(Configuration.class);
    utilizationEndpoint = new UtilizationEndpoint(fileCountBySizeDao, utilizationSchemaDefinition);
    fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition);
    tableCountTask = new TableCountTask(globalStatsDao, sqlConfiguration, reconOMMetadataManager);
    reconScm = (ReconStorageContainerManagerFacade) reconTestInjector.getInstance(OzoneStorageContainerManager.class);
    clusterStateEndpoint = new ClusterStateEndpoint(reconScm, globalStatsDao);
    MetricsServiceProviderFactory metricsServiceProviderFactory = reconTestInjector.getInstance(MetricsServiceProviderFactory.class);
    metricsProxyEndpoint = new MetricsProxyEndpoint(metricsServiceProviderFactory);
    dslContext = getDslContext();
}

Example 5 with ReconStorageContainerManagerFacade

use of org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade in project ozone by apache.

the class TestContainerHealthTask method testDeletedContainer.

@Test
public void testDeletedContainer() throws Exception {
    UnhealthyContainersDao unHealthyContainersTableHandle = getDao(UnhealthyContainersDao.class);
    ContainerHealthSchemaManager containerHealthSchemaManager = new ContainerHealthSchemaManager(getSchemaDefinition(ContainerSchemaDefinition.class), unHealthyContainersTableHandle);
    ReconStorageContainerManagerFacade scmMock = mock(ReconStorageContainerManagerFacade.class);
    MockPlacementPolicy placementMock = new MockPlacementPolicy();
    ContainerManager containerManagerMock = mock(ContainerManager.class);
    StorageContainerServiceProvider scmClientMock = mock(StorageContainerServiceProvider.class);
    // Create 2 containers. The first is OPEN will no replicas, the second is
    // CLOSED with no replicas.
    List<ContainerInfo> mockContainers = getMockContainers(2);
    when(scmMock.getScmServiceProvider()).thenReturn(scmClientMock);
    when(scmMock.getContainerManager()).thenReturn(containerManagerMock);
    when(containerManagerMock.getContainers()).thenReturn(mockContainers);
    for (ContainerInfo c : mockContainers) {
        when(containerManagerMock.getContainer(c.containerID())).thenReturn(c);
        when(scmClientMock.getContainerWithPipeline(c.getContainerID())).thenReturn(new ContainerWithPipeline(c, null));
    }
    // Container State OPEN with no replicas
    when(containerManagerMock.getContainer(ContainerID.valueOf(1L)).getState()).thenReturn(HddsProtos.LifeCycleState.OPEN);
    when(containerManagerMock.getContainerReplicas(ContainerID.valueOf(1L))).thenReturn(Collections.emptySet());
    when(scmClientMock.getContainerWithPipeline(1)).thenReturn(new ContainerWithPipeline(mockContainers.get(0), null));
    // Container State CLOSED with no replicas
    when(containerManagerMock.getContainer(ContainerID.valueOf(2L)).getState()).thenReturn(HddsProtos.LifeCycleState.CLOSED);
    when(containerManagerMock.getContainerReplicas(ContainerID.valueOf(2L))).thenReturn(Collections.emptySet());
    ContainerInfo mockDeletedContainer = getMockDeletedContainer(2);
    when(scmClientMock.getContainerWithPipeline(2)).thenReturn(new ContainerWithPipeline(mockDeletedContainer, null));
    List<UnhealthyContainers> all = unHealthyContainersTableHandle.findAll();
    Assert.assertTrue(all.isEmpty());
    long currentTime = System.currentTimeMillis();
    ReconTaskStatusDao reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
    ReconTaskConfig reconTaskConfig = new ReconTaskConfig();
    reconTaskConfig.setMissingContainerTaskInterval(Duration.ofSeconds(2));
    ContainerHealthTask containerHealthTask = new ContainerHealthTask(scmMock.getContainerManager(), scmMock.getScmServiceProvider(), reconTaskStatusDao, containerHealthSchemaManager, placementMock, reconTaskConfig);
    containerHealthTask.start();
    LambdaTestUtils.await(6000, 1000, () -> (unHealthyContainersTableHandle.count() == 1));
    UnhealthyContainers rec = unHealthyContainersTableHandle.fetchByContainerId(1L).get(0);
    assertEquals("MISSING", rec.getContainerState());
    assertEquals(3, rec.getReplicaDelta().intValue());
    ReconTaskStatus taskStatus = reconTaskStatusDao.findById(containerHealthTask.getTaskName());
    Assert.assertTrue(taskStatus.getLastUpdatedTimestamp() > currentTime);
}