Search in sources :

Example 1 with UnhealthyContainersResponse

use of org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse in project ozone by apache.

the class TestContainerEndpoint method testUnhealthyContainersFilteredResponse.

@Test
public void testUnhealthyContainersFilteredResponse() throws IOException {
    String missing = UnHealthyContainerStates.MISSING.toString();
    Response response = containerEndpoint.getUnhealthyContainers(missing, 1000, 1);
    UnhealthyContainersResponse responseObject = (UnhealthyContainersResponse) response.getEntity();
    assertEquals(0, responseObject.getMissingCount());
    assertEquals(0, responseObject.getOverReplicatedCount());
    assertEquals(0, responseObject.getUnderReplicatedCount());
    assertEquals(0, responseObject.getMisReplicatedCount());
    assertEquals(Collections.EMPTY_LIST, responseObject.getContainers());
    putContainerInfos(5);
    uuid1 = newDatanode("host1", "127.0.0.1");
    uuid2 = newDatanode("host2", "127.0.0.2");
    uuid3 = newDatanode("host3", "127.0.0.3");
    uuid4 = newDatanode("host4", "127.0.0.4");
    createUnhealthyRecords(5, 4, 3, 2);
    response = containerEndpoint.getUnhealthyContainers(missing, 1000, 1);
    responseObject = (UnhealthyContainersResponse) response.getEntity();
    // Summary should have the count for all unhealthy:
    assertEquals(5, responseObject.getMissingCount());
    assertEquals(4, responseObject.getOverReplicatedCount());
    assertEquals(3, responseObject.getUnderReplicatedCount());
    assertEquals(2, responseObject.getMisReplicatedCount());
    Collection<UnhealthyContainerMetadata> records = responseObject.getContainers();
    // There should only be 5 missing containers and no others as we asked for
    // only missing.
    assertEquals(5, records.size());
    for (UnhealthyContainerMetadata r : records) {
        assertEquals(missing, r.getContainerState());
    }
}
Also used : MissingContainersResponse(org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse) KeysResponse(org.apache.hadoop.ozone.recon.api.types.KeysResponse) ContainersResponse(org.apache.hadoop.ozone.recon.api.types.ContainersResponse) Response(javax.ws.rs.core.Response) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) UnhealthyContainerMetadata(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) Test(org.junit.Test)

Example 2 with UnhealthyContainersResponse

use of org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse in project ozone by apache.

the class TestContainerEndpoint method testUnhealthyContainersPaging.

@Test
public void testUnhealthyContainersPaging() throws IOException {
    putContainerInfos(6);
    uuid1 = newDatanode("host1", "127.0.0.1");
    uuid2 = newDatanode("host2", "127.0.0.2");
    uuid3 = newDatanode("host3", "127.0.0.3");
    uuid4 = newDatanode("host4", "127.0.0.4");
    createUnhealthyRecords(5, 4, 3, 2);
    UnhealthyContainersResponse firstBatch = (UnhealthyContainersResponse) containerEndpoint.getUnhealthyContainers(3, 1).getEntity();
    UnhealthyContainersResponse secondBatch = (UnhealthyContainersResponse) containerEndpoint.getUnhealthyContainers(3, 2).getEntity();
    ArrayList<UnhealthyContainerMetadata> records = new ArrayList<>(firstBatch.getContainers());
    assertEquals(3, records.size());
    assertEquals(1L, records.get(0).getContainerID());
    assertEquals(2L, records.get(1).getContainerID());
    assertEquals(3L, records.get(2).getContainerID());
    records = new ArrayList<>(secondBatch.getContainers());
    assertEquals(3, records.size());
    assertEquals(4L, records.get(0).getContainerID());
    assertEquals(5L, records.get(1).getContainerID());
    assertEquals(6L, records.get(2).getContainerID());
}
Also used : UnhealthyContainerMetadata(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata) ArrayList(java.util.ArrayList) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) Test(org.junit.Test)

Example 3 with UnhealthyContainersResponse

use of org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse in project ozone by apache.

the class ContainerEndpoint method getUnhealthyContainers.

/**
 * Return
 * {@link org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata}
 * for all unhealthy containers.
 *
 * @param state Return only containers matching the given unhealthy state,
 *              eg UNDER_REPLICATED, MIS_REPLICATED, OVER_REPLICATED or
 *              MISSING. Passing null returns all containers.
 * @param limit The limit of unhealthy containers to return.
 * @param batchNum The batch number (like "page number") of results to return.
 *                 Passing 1, will return records 1 to limit. 2 will return
 *                 limit + 1 to 2 * limit, etc.
 * @return {@link Response}
 */
@GET
@Path("/unhealthy/{state}")
public Response getUnhealthyContainers(@PathParam("state") String state, @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit, @DefaultValue(DEFAULT_BATCH_NUMBER) @QueryParam(RECON_QUERY_BATCH_PARAM) int batchNum) {
    int offset = Math.max(((batchNum - 1) * limit), 0);
    List<UnhealthyContainerMetadata> unhealthyMeta = new ArrayList<>();
    List<UnhealthyContainersSummary> summary;
    try {
        UnHealthyContainerStates internalState = null;
        if (state != null) {
            // If an invalid state is passed in, this will throw
            // illegalArgumentException and fail the request
            internalState = UnHealthyContainerStates.valueOf(state);
        }
        summary = containerHealthSchemaManager.getUnhealthyContainersSummary();
        List<UnhealthyContainers> containers = containerHealthSchemaManager.getUnhealthyContainers(internalState, offset, limit);
        for (UnhealthyContainers c : containers) {
            long containerID = c.getContainerId();
            ContainerInfo containerInfo = containerManager.getContainer(ContainerID.valueOf(containerID));
            long keyCount = containerInfo.getNumberOfKeys();
            UUID pipelineID = containerInfo.getPipelineID().getId();
            List<ContainerHistory> datanodes = containerManager.getLatestContainerHistory(containerID, containerInfo.getReplicationConfig().getRequiredNodes());
            unhealthyMeta.add(new UnhealthyContainerMetadata(c, datanodes, pipelineID, keyCount));
        }
    } catch (IOException ex) {
        throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);
    } catch (IllegalArgumentException e) {
        throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
    }
    UnhealthyContainersResponse response = new UnhealthyContainersResponse(unhealthyMeta);
    for (UnhealthyContainersSummary s : summary) {
        response.setSummaryCount(s.getContainerState(), s.getCount());
    }
    return Response.ok(response).build();
}
Also used : UnhealthyContainerMetadata(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata) WebApplicationException(javax.ws.rs.WebApplicationException) ArrayList(java.util.ArrayList) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) IOException(java.io.IOException) UnHealthyContainerStates(org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates) ContainerHistory(org.apache.hadoop.ozone.recon.persistence.ContainerHistory) UnhealthyContainers(org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) UUID(java.util.UUID) UnhealthyContainersSummary(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 4 with UnhealthyContainersResponse

use of org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse in project ozone by apache.

the class TestContainerEndpoint method testUnhealthyContainers.

@Test
public void testUnhealthyContainers() throws IOException {
    Response response = containerEndpoint.getUnhealthyContainers(1000, 1);
    UnhealthyContainersResponse responseObject = (UnhealthyContainersResponse) response.getEntity();
    assertEquals(0, responseObject.getMissingCount());
    assertEquals(0, responseObject.getOverReplicatedCount());
    assertEquals(0, responseObject.getUnderReplicatedCount());
    assertEquals(0, responseObject.getMisReplicatedCount());
    assertEquals(Collections.EMPTY_LIST, responseObject.getContainers());
    putContainerInfos(14);
    uuid1 = newDatanode("host1", "127.0.0.1");
    uuid2 = newDatanode("host2", "127.0.0.2");
    uuid3 = newDatanode("host3", "127.0.0.3");
    uuid4 = newDatanode("host4", "127.0.0.4");
    createUnhealthyRecords(5, 4, 3, 2);
    response = containerEndpoint.getUnhealthyContainers(1000, 1);
    responseObject = (UnhealthyContainersResponse) response.getEntity();
    assertEquals(5, responseObject.getMissingCount());
    assertEquals(4, responseObject.getOverReplicatedCount());
    assertEquals(3, responseObject.getUnderReplicatedCount());
    assertEquals(2, responseObject.getMisReplicatedCount());
    Collection<UnhealthyContainerMetadata> records = responseObject.getContainers();
    List<UnhealthyContainerMetadata> missing = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.MISSING.toString())).collect(Collectors.toList());
    assertEquals(5, missing.size());
    assertEquals(3, missing.get(0).getExpectedReplicaCount());
    assertEquals(0, missing.get(0).getActualReplicaCount());
    assertEquals(3, missing.get(0).getReplicaDeltaCount());
    assertEquals(12345L, missing.get(0).getUnhealthySince());
    assertEquals(1L, missing.get(0).getContainerID());
    assertEquals(keyCount, missing.get(0).getKeys());
    assertEquals(pipelineID.getId(), missing.get(0).getPipelineID());
    assertEquals(3, missing.get(0).getReplicas().size());
    assertNull(missing.get(0).getReason());
    Set<String> datanodes = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("host2", "host3", "host4")));
    List<ContainerHistory> containerReplicas = missing.get(0).getReplicas();
    containerReplicas.forEach(history -> {
        Assert.assertTrue(datanodes.contains(history.getDatanodeHost()));
    });
    List<UnhealthyContainerMetadata> overRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.OVER_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(4, overRep.size());
    assertEquals(3, overRep.get(0).getExpectedReplicaCount());
    assertEquals(5, overRep.get(0).getActualReplicaCount());
    assertEquals(-2, overRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, overRep.get(0).getUnhealthySince());
    assertEquals(6L, overRep.get(0).getContainerID());
    assertNull(overRep.get(0).getReason());
    List<UnhealthyContainerMetadata> underRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.UNDER_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(3, underRep.size());
    assertEquals(3, underRep.get(0).getExpectedReplicaCount());
    assertEquals(1, underRep.get(0).getActualReplicaCount());
    assertEquals(2, underRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, underRep.get(0).getUnhealthySince());
    assertEquals(10L, underRep.get(0).getContainerID());
    assertNull(underRep.get(0).getReason());
    List<UnhealthyContainerMetadata> misRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.MIS_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(2, misRep.size());
    assertEquals(2, misRep.get(0).getExpectedReplicaCount());
    assertEquals(1, misRep.get(0).getActualReplicaCount());
    assertEquals(1, misRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, misRep.get(0).getUnhealthySince());
    assertEquals(13L, misRep.get(0).getContainerID());
    assertEquals("some reason", misRep.get(0).getReason());
}
Also used : MissingContainersResponse(org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse) KeysResponse(org.apache.hadoop.ozone.recon.api.types.KeysResponse) ContainersResponse(org.apache.hadoop.ozone.recon.api.types.ContainersResponse) Response(javax.ws.rs.core.Response) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) Arrays(java.util.Arrays) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) MissingContainersResponse(org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse) KeysResponse(org.apache.hadoop.ozone.recon.api.types.KeysResponse) OMMetadataManagerTestUtils.writeDataToOm(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm) StringUtils(org.apache.commons.lang3.StringUtils) BucketLayout(org.apache.hadoop.ozone.om.helpers.BucketLayout) ReconContainerMetadataManager(org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager) StorageContainerServiceProvider(org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) ContainersResponse(org.apache.hadoop.ozone.recon.api.types.ContainersResponse) StorageContainerServiceProviderImpl(org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl) UnHealthyContainerStates(org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates) UnhealthyContainers(org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers) RatisReplicationConfig(org.apache.hadoop.hdds.client.RatisReplicationConfig) OzoneManagerServiceProviderImpl(org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl) ReconContainerManager(org.apache.hadoop.ozone.recon.scm.ReconContainerManager) Collection(java.util.Collection) Set(java.util.Set) ReplicationFactor(org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor) ContainerHealthSchemaManager(org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) ReconOMMetadataManager(org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager) List(java.util.List) Response(javax.ws.rs.core.Response) UnhealthyContainerMetadata(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata) WebApplicationException(javax.ws.rs.WebApplicationException) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) OMMetadataManagerTestUtils.getOmKeyLocationInfo(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getOmKeyLocationInfo) Mockito.mock(org.mockito.Mockito.mock) BlockID(org.apache.hadoop.hdds.client.BlockID) KeyMetadata(org.apache.hadoop.ozone.recon.api.types.KeyMetadata) ArrayList(java.util.ArrayList) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) HashSet(java.util.HashSet) ContainerHistory(org.apache.hadoop.ozone.recon.persistence.ContainerHistory) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) MissingContainerMetadata(org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata) Before(org.junit.Before) OMMetadataManagerTestUtils.getRandomPipeline(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline) Iterator(java.util.Iterator) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) OMMetadataManagerTestUtils.initializeNewOmMetadataManager(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager) ContainerKeyMapperTask(org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask) Assert.assertTrue(org.junit.Assert.assertTrue) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) IOException(java.io.IOException) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) ContainerSchemaDefinition(org.hadoop.ozone.recon.schema.ContainerSchemaDefinition) ContainerMetadata(org.apache.hadoop.ozone.recon.api.types.ContainerMetadata) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) OMMetadataManagerTestUtils.getTestReconOmMetadataManager(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager) Assert.assertNull(org.junit.Assert.assertNull) Rule(org.junit.Rule) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) Table(org.apache.hadoop.hdds.utils.db.Table) PipelineID(org.apache.hadoop.hdds.scm.pipeline.PipelineID) ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) ReconPipelineManager(org.apache.hadoop.ozone.recon.scm.ReconPipelineManager) ReconTestInjector(org.apache.hadoop.ozone.recon.ReconTestInjector) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) TemporaryFolder(org.junit.rules.TemporaryFolder) OMMetadataManager(org.apache.hadoop.ozone.om.OMMetadataManager) UnhealthyContainerMetadata(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata) ContainerHistory(org.apache.hadoop.ozone.recon.persistence.ContainerHistory) UnhealthyContainersResponse(org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)3 UnhealthyContainerMetadata (org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata)3 UnhealthyContainersResponse (org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse)3 IOException (java.io.IOException)2 UUID (java.util.UUID)2 WebApplicationException (javax.ws.rs.WebApplicationException)2 Response (javax.ws.rs.core.Response)2 ContainerInfo (org.apache.hadoop.hdds.scm.container.ContainerInfo)2 ContainersResponse (org.apache.hadoop.ozone.recon.api.types.ContainersResponse)2 Test (org.junit.Test)2 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 GET (javax.ws.rs.GET)1