Examples with ApplicationReport org.apache.hadoop.yarn.api.records.ApplicationReport used on opensource projects

Example 16 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestYarnClient method testGetApplicationAttempt.

@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final YarnClient client = new MockYarnClient();
    client.init(conf);
    client.start();
    List<ApplicationReport> expectedReports = ((MockYarnClient) client).getReports();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ApplicationAttemptReport report = client.getApplicationAttemptReport(appAttemptId);
    Assert.assertNotNull(report);
    Assert.assertEquals(report.getApplicationAttemptId().toString(), expectedReports.get(0).getCurrentApplicationAttemptId().toString());
    client.stop();
}

Example 17 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class RMWebServices method getApps.

@GET
@Path("/apps")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("state") String stateQuery, @QueryParam("states") Set<String> statesQuery, @QueryParam("finalStatus") String finalStatusQuery, @QueryParam("user") String userQuery, @QueryParam("queue") String queueQuery, @QueryParam("limit") String count, @QueryParam("startedTimeBegin") String startedBegin, @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, @QueryParam("applicationTypes") Set<String> applicationTypes, @QueryParam("applicationTags") Set<String> applicationTags) {
    boolean checkCount = false;
    boolean checkStart = false;
    boolean checkEnd = false;
    boolean checkAppTypes = false;
    boolean checkAppStates = false;
    boolean checkAppTags = false;
    long countNum = 0;
    // set values suitable in case both of begin/end not specified
    long sBegin = 0;
    long sEnd = Long.MAX_VALUE;
    long fBegin = 0;
    long fEnd = Long.MAX_VALUE;
    init();
    if (count != null && !count.isEmpty()) {
        checkCount = true;
        countNum = Long.parseLong(count);
        if (countNum <= 0) {
            throw new BadRequestException("limit value must be greater then 0");
        }
    }
    if (startedBegin != null && !startedBegin.isEmpty()) {
        checkStart = true;
        sBegin = Long.parseLong(startedBegin);
        if (sBegin < 0) {
            throw new BadRequestException("startedTimeBegin must be greater than 0");
        }
    }
    if (startedEnd != null && !startedEnd.isEmpty()) {
        checkStart = true;
        sEnd = Long.parseLong(startedEnd);
        if (sEnd < 0) {
            throw new BadRequestException("startedTimeEnd must be greater than 0");
        }
    }
    if (sBegin > sEnd) {
        throw new BadRequestException("startedTimeEnd must be greater than startTimeBegin");
    }
    if (finishBegin != null && !finishBegin.isEmpty()) {
        checkEnd = true;
        fBegin = Long.parseLong(finishBegin);
        if (fBegin < 0) {
            throw new BadRequestException("finishTimeBegin must be greater than 0");
        }
    }
    if (finishEnd != null && !finishEnd.isEmpty()) {
        checkEnd = true;
        fEnd = Long.parseLong(finishEnd);
        if (fEnd < 0) {
            throw new BadRequestException("finishTimeEnd must be greater than 0");
        }
    }
    if (fBegin > fEnd) {
        throw new BadRequestException("finishTimeEnd must be greater than finishTimeBegin");
    }
    Set<String> appTypes = parseQueries(applicationTypes, false);
    if (!appTypes.isEmpty()) {
        checkAppTypes = true;
    }
    Set<String> appTags = parseQueries(applicationTags, false);
    if (!appTags.isEmpty()) {
        checkAppTags = true;
    }
    // stateQuery is deprecated.
    if (stateQuery != null && !stateQuery.isEmpty()) {
        statesQuery.add(stateQuery);
    }
    Set<String> appStates = parseQueries(statesQuery, true);
    if (!appStates.isEmpty()) {
        checkAppStates = true;
    }
    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    if (checkStart) {
        request.setStartRange(sBegin, sEnd);
    }
    if (checkEnd) {
        request.setFinishRange(fBegin, fEnd);
    }
    if (checkCount) {
        request.setLimit(countNum);
    }
    if (checkAppTypes) {
        request.setApplicationTypes(appTypes);
    }
    if (checkAppTags) {
        request.setApplicationTags(appTags);
    }
    if (checkAppStates) {
        request.setApplicationStates(appStates);
    }
    if (queueQuery != null && !queueQuery.isEmpty()) {
        ResourceScheduler rs = rm.getResourceScheduler();
        if (rs instanceof CapacityScheduler) {
            CapacityScheduler cs = (CapacityScheduler) rs;
            // validate queue exists
            try {
                cs.getQueueInfo(queueQuery, false, false);
            } catch (IOException e) {
                throw new BadRequestException(e.getMessage());
            }
        }
        Set<String> queues = new HashSet<String>(1);
        queues.add(queueQuery);
        request.setQueues(queues);
    }
    if (userQuery != null && !userQuery.isEmpty()) {
        Set<String> users = new HashSet<String>(1);
        users.add(userQuery);
        request.setUsers(users);
    }
    List<ApplicationReport> appReports = null;
    try {
        appReports = rm.getClientRMService().getApplications(request, false).getApplicationList();
    } catch (YarnException e) {
        LOG.error("Unable to retrieve apps from ClientRMService", e);
        throw new YarnRuntimeException("Unable to retrieve apps from ClientRMService", e);
    }
    final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext().getRMApps();
    AppsInfo allApps = new AppsInfo();
    for (ApplicationReport report : appReports) {
        RMApp rmapp = apps.get(report.getApplicationId());
        if (rmapp == null) {
            continue;
        }
        if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
            FinalApplicationStatus.valueOf(finalStatusQuery);
            if (!rmapp.getFinalApplicationStatus().toString().equalsIgnoreCase(finalStatusQuery)) {
                continue;
            }
        }
        AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf));
        allApps.add(app);
    }
    return allApps;
}

Example 18 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class RMAppsBlock method renderData.

@Override
protected void renderData(Block html) {
    TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User").th(".name", "Name").th(".type", "Application Type").th(".queue", "Queue").th(".priority", "Application Priority").th(".starttime", "StartTime").th(".finishtime", "FinishTime").th(".state", "State").th(".finalstatus", "FinalStatus").th(".runningcontainer", "Running Containers").th(".allocatedCpu", "Allocated CPU VCores").th(".allocatedMemory", "Allocated Memory MB").th(".queuePercentage", "% of Queue").th(".clusterPercentage", "% of Cluster").th(".progress", "Progress").th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()._().tbody();
    StringBuilder appsTableData = new StringBuilder("[\n");
    for (ApplicationReport appReport : appReports) {
        // hasn't filtering capability (YARN-1819).
        if (!reqAppStates.isEmpty() && !reqAppStates.contains(appReport.getYarnApplicationState())) {
            continue;
        }
        AppInfo app = new AppInfo(appReport);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(app.getCurrentAppAttemptId());
        String queuePercent = "N/A";
        String clusterPercent = "N/A";
        if (appReport.getApplicationResourceUsageReport() != null) {
            queuePercent = String.format("%.1f", appReport.getApplicationResourceUsageReport().getQueueUsagePercentage());
            clusterPercent = String.format("%.1f", appReport.getApplicationResourceUsageReport().getClusterUsagePercentage());
        }
        String blacklistedNodesCount = "N/A";
        RMAppAttempt appAttempt = rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()).getAppAttempts().get(appAttemptId);
        Set<String> nodes = null == appAttempt ? null : appAttempt.getBlacklistedNodes();
        if (nodes != null) {
            blacklistedNodesCount = String.valueOf(nodes.size());
        }
        String percent = StringUtils.format("%.1f", app.getProgress());
        appsTableData.append("[\"<a href='").append(url("app", app.getAppId())).append("'>").append(app.getAppId()).append("</a>\",\"").append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app.getUser()))).append("\",\"").append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app.getName()))).append("\",\"").append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app.getType()))).append("\",\"").append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app.getQueue()))).append("\",\"").append(String.valueOf(app.getPriority())).append("\",\"").append(app.getStartedTime()).append("\",\"").append(app.getFinishedTime()).append("\",\"").append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()).append("\",\"").append(app.getFinalAppStatus()).append("\",\"").append(app.getRunningContainers() == -1 ? "N/A" : String.valueOf(app.getRunningContainers())).append("\",\"").append(app.getAllocatedCpuVcores() == -1 ? "N/A" : String.valueOf(app.getAllocatedCpuVcores())).append("\",\"").append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getAllocatedMemoryMB())).append("\",\"").append(queuePercent).append("\",\"").append(clusterPercent).append("\",\"").append("<br title='").append(percent).append("'> <div class='").append(C_PROGRESSBAR).append("' title='").append(join(percent, '%')).append("'> ").append("<div class='").append(C_PROGRESSBAR_VALUE).append("' style='").append(join("width:", percent, '%')).append("'> </div> </div>").append("\",\"<a ");
        String trackingURL = app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) || app.getAppState() == YarnApplicationState.NEW ? null : app.getTrackingUrl();
        String trackingUI = app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) || app.getAppState() == YarnApplicationState.NEW ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED || app.getAppState() == YarnApplicationState.FAILED || app.getAppState() == YarnApplicationState.KILLED ? "History" : "ApplicationMaster";
        appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL).append("'>").append(trackingUI).append("</a>\",").append("\"").append(blacklistedNodesCount).append("\"],\n");
    }
    if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
        appsTableData.delete(appsTableData.length() - 2, appsTableData.length() - 1);
    }
    appsTableData.append("]");
    html.script().$type("text/javascript")._("var appsTableData=" + appsTableData)._();
    tbody._()._();
}

Example 19 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class ClientServiceDelegate method getProxy.

private MRClientProtocol getProxy() throws IOException {
    if (realProxy != null) {
        return realProxy;
    }
    // Possibly allow nulls through the PB tunnel, otherwise deal with an exception
    // and redirect to the history server.
    ApplicationReport application = null;
    try {
        application = rm.getApplicationReport(appId);
    } catch (ApplicationNotFoundException e) {
        application = null;
    } catch (YarnException e2) {
        throw new IOException(e2);
    }
    if (application != null) {
        trackingUrl = application.getTrackingUrl();
    }
    InetSocketAddress serviceAddr = null;
    while (application == null || YarnApplicationState.RUNNING == application.getYarnApplicationState()) {
        if (application == null) {
            LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
            return checkAndGetHSProxy(null, JobState.NEW);
        }
        try {
            if (application.getHost() == null || "".equals(application.getHost())) {
                LOG.debug("AM not assigned to Job. Waiting to get the AM ...");
                Thread.sleep(2000);
                LOG.debug("Application state is " + application.getYarnApplicationState());
                application = rm.getApplicationReport(appId);
                continue;
            } else if (UNAVAILABLE.equals(application.getHost())) {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(application, JobState.RUNNING);
            }
            if (!conf.getBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, false)) {
                UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName());
                serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort());
                if (UserGroupInformation.isSecurityEnabled()) {
                    org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
                    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
                    newUgi.addToken(token);
                }
                LOG.debug("Connecting to " + serviceAddr);
                final InetSocketAddress finalServiceAddr = serviceAddr;
                realProxy = newUgi.doAs(new PrivilegedExceptionAction<MRClientProtocol>() {

                    @Override
                    public MRClientProtocol run() throws IOException {
                        return instantiateAMProxy(finalServiceAddr);
                    }
                });
            } else {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(null, JobState.RUNNING);
            }
            return realProxy;
        } catch (IOException e) {
            //possibly the AM has crashed
            //there may be some time before AM is restarted
            //keep retrying by getting the address from RM
            LOG.info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address...");
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e1) {
                LOG.warn("getProxy() call interruped", e1);
                throw new YarnRuntimeException(e1);
            }
            try {
                application = rm.getApplicationReport(appId);
            } catch (YarnException e1) {
                throw new IOException(e1);
            }
            if (application == null) {
                LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
                return checkAndGetHSProxy(null, JobState.RUNNING);
            }
        } catch (InterruptedException e) {
            LOG.warn("getProxy() call interruped", e);
            throw new YarnRuntimeException(e);
        } catch (YarnException e) {
            throw new IOException(e);
        }
    }
    /** we just want to return if its allocating, so that we don't
     * block on it. This is to be able to return job status
     * on an allocating Application.
     */
    String user = application.getUser();
    if (user == null) {
        throw new IOException("User is not set in the application report");
    }
    if (application.getYarnApplicationState() == YarnApplicationState.NEW || application.getYarnApplicationState() == YarnApplicationState.NEW_SAVING || application.getYarnApplicationState() == YarnApplicationState.SUBMITTED || application.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.NEW);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FAILED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.FAILED);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.KILLED);
    }
    //succeeded.
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED) {
        LOG.info("Application state is completed. FinalApplicationStatus=" + application.getFinalApplicationStatus().toString() + ". Redirecting to job history server");
        realProxy = checkAndGetHSProxy(application, JobState.SUCCEEDED);
    }
    return realProxy;
}

Example 20 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class YARNRunner method killUnFinishedApplication.

private void killUnFinishedApplication(ApplicationId appId) throws IOException {
    ApplicationReport application = null;
    try {
        application = resMgrDelegate.getApplicationReport(appId);
    } catch (YarnException e) {
        throw new IOException(e);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED || application.getYarnApplicationState() == YarnApplicationState.FAILED || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        return;
    }
    killApplication(appId);
}