Examples with ContainerReport org.apache.hadoop.yarn.api.records.ContainerReport used on opensource projects

Example 16 with ContainerReport

use of org.apache.hadoop.yarn.api.records.ContainerReport in project samza by apache.

the class YarnJobValidationTool method validateContainerCount.

public int validateContainerCount(ApplicationAttemptId attemptId) throws Exception {
    int runningContainerCount = 0;
    for (ContainerReport containerReport : this.client.getContainers(attemptId)) {
        if (containerReport.getContainerState() == ContainerState.RUNNING) {
            ++runningContainerCount;
        }
    }
    // expected containers to be the configured job containers plus the AppMaster container
    int containerExpected = this.config.getContainerCount() + 1;
    if (runningContainerCount == containerExpected) {
        log.info("Container count matches. " + runningContainerCount + " containers are running.");
        return runningContainerCount;
    } else {
        throw new SamzaException("Container count does not match. " + runningContainerCount + " containers are running, while " + containerExpected + " is expected.");
    }
}

Example 17 with ContainerReport

use of org.apache.hadoop.yarn.api.records.ContainerReport in project samza by apache.

the class TestYarnJobValidationTool method testValidateContainerCount.

@Test
public void testValidateContainerCount() throws Exception {
    List<ContainerReport> containerReports = new ArrayList<>();
    for (int i = 0; i <= containerCount; i++) {
        ContainerReport report = mock(ContainerReport.class);
        when(report.getContainerState()).thenReturn(ContainerState.RUNNING);
        containerReports.add(report);
    }
    when(client.getContainers(attemptId)).thenReturn(containerReports);
    assertTrue(tool.validateContainerCount(attemptId) == (containerCount + 1));
    containerReports.remove(0);
    exception.expect(SamzaException.class);
    tool.validateContainerCount(attemptId);
}

Example 18 with ContainerReport

use of org.apache.hadoop.yarn.api.records.ContainerReport in project hadoop by apache.

the class YarnClientImpl method getContainers.

@Override
public List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId) throws YarnException, IOException {
    List<ContainerReport> containersForAttempt = new ArrayList<ContainerReport>();
    boolean appNotFoundInRM = false;
    try {
        GetContainersRequest request = Records.newRecord(GetContainersRequest.class);
        request.setApplicationAttemptId(applicationAttemptId);
        GetContainersResponse response = rmClient.getContainers(request);
        containersForAttempt.addAll(response.getContainerList());
    } catch (YarnException e) {
        if (e.getClass() != ApplicationNotFoundException.class || !historyServiceEnabled) {
            // need to check with history service else throw exception.
            throw e;
        }
        appNotFoundInRM = true;
    }
    if (historyServiceEnabled) {
        // Check with AHS even if found in RM because to capture info of finished
        // containers also
        List<ContainerReport> containersListFromAHS = null;
        try {
            containersListFromAHS = historyClient.getContainers(applicationAttemptId);
        } catch (IOException e) {
            // is disabled hence app not found exception is possible
            if (appNotFoundInRM) {
                // app not found in bothM and RM then propagate the exception.
                throw e;
            }
        }
        if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
            // remove duplicates
            Set<ContainerId> containerIdsToBeKeptFromAHS = new HashSet<ContainerId>();
            Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
            while (tmpItr.hasNext()) {
                containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
            }
            Iterator<ContainerReport> rmContainers = containersForAttempt.iterator();
            while (rmContainers.hasNext()) {
                ContainerReport tmp = rmContainers.next();
                containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
            // Remove containers from AHS as container from RM will have latest
            // information
            }
            if (containerIdsToBeKeptFromAHS.size() > 0 && containersListFromAHS.size() != containerIdsToBeKeptFromAHS.size()) {
                Iterator<ContainerReport> containersFromHS = containersListFromAHS.iterator();
                while (containersFromHS.hasNext()) {
                    ContainerReport containerReport = containersFromHS.next();
                    if (containerIdsToBeKeptFromAHS.contains(containerReport.getContainerId())) {
                        containersForAttempt.add(containerReport);
                    }
                }
            } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS.size()) {
                containersForAttempt.addAll(containersListFromAHS);
            }
        }
    }
    return containersForAttempt;
}

Example 19 with ContainerReport

use of org.apache.hadoop.yarn.api.records.ContainerReport in project hadoop by apache.

the class LogsCLI method getContainerReportsFromRunningApplication.

private List<ContainerReport> getContainerReportsFromRunningApplication(ContainerLogsRequest options) throws YarnException, IOException {
    List<ContainerReport> reports = new ArrayList<ContainerReport>();
    List<ApplicationAttemptReport> attempts = yarnClient.getApplicationAttempts(options.getAppId());
    Map<ContainerId, ContainerReport> containerMap = new TreeMap<ContainerId, ContainerReport>();
    for (ApplicationAttemptReport attempt : attempts) {
        List<ContainerReport> containers = yarnClient.getContainers(attempt.getApplicationAttemptId());
        for (ContainerReport container : containers) {
            if (!containerMap.containsKey(container.getContainerId())) {
                containerMap.put(container.getContainerId(), container);
            }
        }
    }
    reports.addAll(containerMap.values());
    return reports;
}

Example 20 with ContainerReport

use of org.apache.hadoop.yarn.api.records.ContainerReport in project hadoop by apache.

the class LogsCLI method getContainersLogRequestForRunningApplication.

private List<ContainerLogsRequest> getContainersLogRequestForRunningApplication(ContainerLogsRequest options) throws YarnException, IOException {
    List<ContainerLogsRequest> newOptionsList = new ArrayList<ContainerLogsRequest>();
    List<ContainerReport> reports = getContainerReportsFromRunningApplication(options);
    for (ContainerReport container : reports) {
        ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
        newOptions.setContainerId(container.getContainerId().toString());
        newOptions.setNodeId(container.getAssignedNode().toString());
        String httpAddress = container.getNodeHttpAddress();
        if (httpAddress != null && !httpAddress.isEmpty()) {
            newOptions.setNodeHttpAddress(httpAddress.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
        }
        newOptions.setContainerState(container.getContainerState());
        newOptionsList.add(newOptions);
    }
    return newOptionsList;
}