Examples with ContainerHistory org.apache.hadoop.ozone.recon.persistence.ContainerHistory used on opensource projects

Example 1 with ContainerHistory

use of org.apache.hadoop.ozone.recon.persistence.ContainerHistory in project ozone by apache.

the class TestContainerEndpoint method testGetReplicaHistoryForContainer.

@Test
public void testGetReplicaHistoryForContainer() throws IOException {
    // Add container history for container id 1
    final UUID u1 = newDatanode("host1", "127.0.0.1");
    final UUID u2 = newDatanode("host2", "127.0.0.2");
    final UUID u3 = newDatanode("host3", "127.0.0.3");
    final UUID u4 = newDatanode("host4", "127.0.0.4");
    reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L);
    reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L);
    reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L);
    reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L);
    reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L);
    Response response = containerEndpoint.getReplicaHistoryForContainer(1L);
    List<ContainerHistory> histories = (List<ContainerHistory>) response.getEntity();
    Set<String> datanodes = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(u1.toString(), u2.toString(), u3.toString(), u4.toString())));
    Assert.assertEquals(4, histories.size());
    histories.forEach(history -> {
        Assert.assertTrue(datanodes.contains(history.getDatanodeUuid()));
        if (history.getDatanodeUuid().equals(u1.toString())) {
            Assert.assertEquals("host1", history.getDatanodeHost());
            Assert.assertEquals(1L, history.getFirstSeenTime());
            Assert.assertEquals(5L, history.getLastSeenTime());
        }
    });
    // Check getLatestContainerHistory
    List<ContainerHistory> hist1 = reconContainerManager.getLatestContainerHistory(1L, 10);
    Assert.assertTrue(hist1.size() <= 10);
    // Descending order by last report timestamp
    for (int i = 0; i < hist1.size() - 1; i++) {
        Assert.assertTrue(hist1.get(i).getLastSeenTime() >= hist1.get(i + 1).getLastSeenTime());
    }
}

Example 2 with ContainerHistory

use of org.apache.hadoop.ozone.recon.persistence.ContainerHistory in project ozone by apache.

the class ContainerEndpoint method getUnhealthyContainers.

/**
 * Return
 * {@link org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata}
 * for all unhealthy containers.
 *
 * @param state Return only containers matching the given unhealthy state,
 *              eg UNDER_REPLICATED, MIS_REPLICATED, OVER_REPLICATED or
 *              MISSING. Passing null returns all containers.
 * @param limit The limit of unhealthy containers to return.
 * @param batchNum The batch number (like "page number") of results to return.
 *                 Passing 1, will return records 1 to limit. 2 will return
 *                 limit + 1 to 2 * limit, etc.
 * @return {@link Response}
 */
@GET
@Path("/unhealthy/{state}")
public Response getUnhealthyContainers(@PathParam("state") String state, @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit, @DefaultValue(DEFAULT_BATCH_NUMBER) @QueryParam(RECON_QUERY_BATCH_PARAM) int batchNum) {
    int offset = Math.max(((batchNum - 1) * limit), 0);
    List<UnhealthyContainerMetadata> unhealthyMeta = new ArrayList<>();
    List<UnhealthyContainersSummary> summary;
    try {
        UnHealthyContainerStates internalState = null;
        if (state != null) {
            // If an invalid state is passed in, this will throw
            // illegalArgumentException and fail the request
            internalState = UnHealthyContainerStates.valueOf(state);
        }
        summary = containerHealthSchemaManager.getUnhealthyContainersSummary();
        List<UnhealthyContainers> containers = containerHealthSchemaManager.getUnhealthyContainers(internalState, offset, limit);
        for (UnhealthyContainers c : containers) {
            long containerID = c.getContainerId();
            ContainerInfo containerInfo = containerManager.getContainer(ContainerID.valueOf(containerID));
            long keyCount = containerInfo.getNumberOfKeys();
            UUID pipelineID = containerInfo.getPipelineID().getId();
            List<ContainerHistory> datanodes = containerManager.getLatestContainerHistory(containerID, containerInfo.getReplicationConfig().getRequiredNodes());
            unhealthyMeta.add(new UnhealthyContainerMetadata(c, datanodes, pipelineID, keyCount));
        }
    } catch (IOException ex) {
        throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);
    } catch (IllegalArgumentException e) {
        throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
    }
    UnhealthyContainersResponse response = new UnhealthyContainersResponse(unhealthyMeta);
    for (UnhealthyContainersSummary s : summary) {
        response.setSummaryCount(s.getContainerState(), s.getCount());
    }
    return Response.ok(response).build();
}

Example 3 with ContainerHistory

use of org.apache.hadoop.ozone.recon.persistence.ContainerHistory in project ozone by apache.

the class ReconContainerManager method getAllContainerHistory.

public List<ContainerHistory> getAllContainerHistory(long containerID) {
    // First, get the existing entries from DB
    Map<UUID, ContainerReplicaHistory> resMap;
    try {
        resMap = cdbServiceProvider.getContainerReplicaHistory(containerID);
    } catch (IOException ex) {
        resMap = new HashMap<>();
        LOG.debug("Unable to retrieve container replica history from RDB.");
    }
    // Then, update the entries with the latest in-memory info, if available
    if (replicaHistoryMap != null) {
        Map<UUID, ContainerReplicaHistory> replicaLastSeenMap = replicaHistoryMap.get(containerID);
        if (replicaLastSeenMap != null) {
            Map<UUID, ContainerReplicaHistory> finalResMap = resMap;
            replicaLastSeenMap.forEach((k, v) -> finalResMap.merge(k, v, (old, latest) -> latest));
            resMap = finalResMap;
        }
    }
    // Finally, convert map to list for output
    List<ContainerHistory> resList = new ArrayList<>();
    for (Map.Entry<UUID, ContainerReplicaHistory> entry : resMap.entrySet()) {
        final UUID uuid = entry.getKey();
        String hostname = "N/A";
        // Attempt to retrieve hostname from NODES table
        if (nodeDB != null) {
            try {
                DatanodeDetails dnDetails = nodeDB.get(uuid);
                if (dnDetails != null) {
                    hostname = dnDetails.getHostName();
                }
            } catch (IOException ex) {
                LOG.debug("Unable to retrieve from NODES table of node {}. {}", uuid, ex.getMessage());
            }
        }
        final long firstSeenTime = entry.getValue().getFirstSeenTime();
        final long lastSeenTime = entry.getValue().getLastSeenTime();
        long bcsId = entry.getValue().getBcsId();
        resList.add(new ContainerHistory(containerID, uuid.toString(), hostname, firstSeenTime, lastSeenTime, bcsId));
    }
    return resList;
}

Example 4 with ContainerHistory

use of org.apache.hadoop.ozone.recon.persistence.ContainerHistory in project ozone by apache.

the class ContainerEndpoint method getMissingContainers.

/**
 * Return
 * {@link org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata}
 * for all missing containers.
 *
 * @return {@link Response}
 */
@GET
@Path("/missing")
public Response getMissingContainers() {
    List<MissingContainerMetadata> missingContainers = new ArrayList<>();
    containerHealthSchemaManager.getUnhealthyContainers(UnHealthyContainerStates.MISSING, 0, Integer.MAX_VALUE).forEach(container -> {
        long containerID = container.getContainerId();
        try {
            ContainerInfo containerInfo = containerManager.getContainer(ContainerID.valueOf(containerID));
            long keyCount = containerInfo.getNumberOfKeys();
            UUID pipelineID = containerInfo.getPipelineID().getId();
            List<ContainerHistory> datanodes = containerManager.getLatestContainerHistory(containerID, containerInfo.getReplicationConfig().getRequiredNodes());
            missingContainers.add(new MissingContainerMetadata(containerID, container.getInStateSince(), keyCount, pipelineID, datanodes));
        } catch (IOException ioEx) {
            throw new WebApplicationException(ioEx, Response.Status.INTERNAL_SERVER_ERROR);
        }
    });
    MissingContainersResponse response = new MissingContainersResponse(missingContainers.size(), missingContainers);
    return Response.ok(response).build();
}

Example 5 with ContainerHistory

use of org.apache.hadoop.ozone.recon.persistence.ContainerHistory in project ozone by apache.

the class TestContainerEndpoint method testUnhealthyContainers.

@Test
public void testUnhealthyContainers() throws IOException {
    Response response = containerEndpoint.getUnhealthyContainers(1000, 1);
    UnhealthyContainersResponse responseObject = (UnhealthyContainersResponse) response.getEntity();
    assertEquals(0, responseObject.getMissingCount());
    assertEquals(0, responseObject.getOverReplicatedCount());
    assertEquals(0, responseObject.getUnderReplicatedCount());
    assertEquals(0, responseObject.getMisReplicatedCount());
    assertEquals(Collections.EMPTY_LIST, responseObject.getContainers());
    putContainerInfos(14);
    uuid1 = newDatanode("host1", "127.0.0.1");
    uuid2 = newDatanode("host2", "127.0.0.2");
    uuid3 = newDatanode("host3", "127.0.0.3");
    uuid4 = newDatanode("host4", "127.0.0.4");
    createUnhealthyRecords(5, 4, 3, 2);
    response = containerEndpoint.getUnhealthyContainers(1000, 1);
    responseObject = (UnhealthyContainersResponse) response.getEntity();
    assertEquals(5, responseObject.getMissingCount());
    assertEquals(4, responseObject.getOverReplicatedCount());
    assertEquals(3, responseObject.getUnderReplicatedCount());
    assertEquals(2, responseObject.getMisReplicatedCount());
    Collection<UnhealthyContainerMetadata> records = responseObject.getContainers();
    List<UnhealthyContainerMetadata> missing = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.MISSING.toString())).collect(Collectors.toList());
    assertEquals(5, missing.size());
    assertEquals(3, missing.get(0).getExpectedReplicaCount());
    assertEquals(0, missing.get(0).getActualReplicaCount());
    assertEquals(3, missing.get(0).getReplicaDeltaCount());
    assertEquals(12345L, missing.get(0).getUnhealthySince());
    assertEquals(1L, missing.get(0).getContainerID());
    assertEquals(keyCount, missing.get(0).getKeys());
    assertEquals(pipelineID.getId(), missing.get(0).getPipelineID());
    assertEquals(3, missing.get(0).getReplicas().size());
    assertNull(missing.get(0).getReason());
    Set<String> datanodes = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("host2", "host3", "host4")));
    List<ContainerHistory> containerReplicas = missing.get(0).getReplicas();
    containerReplicas.forEach(history -> {
        Assert.assertTrue(datanodes.contains(history.getDatanodeHost()));
    });
    List<UnhealthyContainerMetadata> overRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.OVER_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(4, overRep.size());
    assertEquals(3, overRep.get(0).getExpectedReplicaCount());
    assertEquals(5, overRep.get(0).getActualReplicaCount());
    assertEquals(-2, overRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, overRep.get(0).getUnhealthySince());
    assertEquals(6L, overRep.get(0).getContainerID());
    assertNull(overRep.get(0).getReason());
    List<UnhealthyContainerMetadata> underRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.UNDER_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(3, underRep.size());
    assertEquals(3, underRep.get(0).getExpectedReplicaCount());
    assertEquals(1, underRep.get(0).getActualReplicaCount());
    assertEquals(2, underRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, underRep.get(0).getUnhealthySince());
    assertEquals(10L, underRep.get(0).getContainerID());
    assertNull(underRep.get(0).getReason());
    List<UnhealthyContainerMetadata> misRep = records.stream().filter(r -> r.getContainerState().equals(UnHealthyContainerStates.MIS_REPLICATED.toString())).collect(Collectors.toList());
    assertEquals(2, misRep.size());
    assertEquals(2, misRep.get(0).getExpectedReplicaCount());
    assertEquals(1, misRep.get(0).getActualReplicaCount());
    assertEquals(1, misRep.get(0).getReplicaDeltaCount());
    assertEquals(12345L, misRep.get(0).getUnhealthySince());
    assertEquals(13L, misRep.get(0).getContainerID());
    assertEquals("some reason", misRep.get(0).getReason());
}