Examples with GetApplicationsRequest org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest used on opensource projects

Example 16 with GetApplicationsRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest in project hadoop by apache.

the class TestRMRestart method testRMRestartGetApplicationList.

@Test(timeout = 60000)
public void testRMRestartGetApplicationList() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm1);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // a succeeded app.
    RMApp app0 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am0 = launchAM(app0, rm1, nm1);
    finishApplicationMaster(app0, rm1, nm1, am0);
    // a failed app.
    RMApp app1 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am1 = launchAM(app1, rm1, nm1);
    // fail the AM by sending CONTAINER_FINISHED event without registering.
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED);
    // a killed app.
    RMApp app2 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am2 = launchAM(app2, rm1, nm1);
    rm1.killApp(app2.getApplicationId());
    rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    verify(rm1.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    // restart rm
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        protected RMAppManager createRMAppManager() {
            return spy(super.createRMAppManager());
        }

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm2);
    rm2.start();
    verify(rm2.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    GetApplicationsRequest request1 = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED));
    GetApplicationsResponse response1 = rm2.getClientRMService().getApplications(request1);
    List<ApplicationReport> appList1 = response1.getApplicationList();
    // assert all applications exist according to application state after RM
    // restarts.
    boolean forApp0 = false, forApp1 = false, forApp2 = false;
    for (ApplicationReport report : appList1) {
        if (report.getApplicationId().equals(app0.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FINISHED, report.getYarnApplicationState());
            forApp0 = true;
        }
        if (report.getApplicationId().equals(app1.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FAILED, report.getYarnApplicationState());
            forApp1 = true;
        }
        if (report.getApplicationId().equals(app2.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.KILLED, report.getYarnApplicationState());
            forApp2 = true;
        }
    }
    Assert.assertTrue(forApp0 && forApp1 && forApp2);
    // assert all applications exist according to application type after RM
    // restarts.
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("myType");
    GetApplicationsRequest request2 = GetApplicationsRequest.newInstance(appTypes);
    GetApplicationsResponse response2 = rm2.getClientRMService().getApplications(request2);
    List<ApplicationReport> appList2 = response2.getApplicationList();
    Assert.assertTrue(3 == appList2.size());
    // check application summary is logged for the completed apps with timeout
    // to make sure APP_COMPLETED events are processed, after RM restart.
    verify(rm2.getRMAppManager(), timeout(1000).times(3)).logApplicationSummary(isA(ApplicationId.class));
}

Example 17 with GetApplicationsRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest in project hadoop by apache.

the class WebServices method getApps.

public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, String stateQuery, Set<String> statesQuery, String finalStatusQuery, String userQuery, String queueQuery, String count, String startedBegin, String startedEnd, String finishBegin, String finishEnd, Set<String> applicationTypes) {
    UserGroupInformation callerUGI = getUser(req);
    boolean checkEnd = false;
    boolean checkAppTypes = false;
    boolean checkAppStates = false;
    long countNum = Long.MAX_VALUE;
    // set values suitable in case both of begin/end not specified
    long sBegin = 0;
    long sEnd = Long.MAX_VALUE;
    long fBegin = 0;
    long fEnd = Long.MAX_VALUE;
    if (count != null && !count.isEmpty()) {
        countNum = Long.parseLong(count);
        if (countNum <= 0) {
            throw new BadRequestException("limit value must be greater then 0");
        }
    }
    if (startedBegin != null && !startedBegin.isEmpty()) {
        sBegin = Long.parseLong(startedBegin);
        if (sBegin < 0) {
            throw new BadRequestException("startedTimeBegin must be greater than 0");
        }
    }
    if (startedEnd != null && !startedEnd.isEmpty()) {
        sEnd = Long.parseLong(startedEnd);
        if (sEnd < 0) {
            throw new BadRequestException("startedTimeEnd must be greater than 0");
        }
    }
    if (sBegin > sEnd) {
        throw new BadRequestException("startedTimeEnd must be greater than startTimeBegin");
    }
    if (finishBegin != null && !finishBegin.isEmpty()) {
        checkEnd = true;
        fBegin = Long.parseLong(finishBegin);
        if (fBegin < 0) {
            throw new BadRequestException("finishTimeBegin must be greater than 0");
        }
    }
    if (finishEnd != null && !finishEnd.isEmpty()) {
        checkEnd = true;
        fEnd = Long.parseLong(finishEnd);
        if (fEnd < 0) {
            throw new BadRequestException("finishTimeEnd must be greater than 0");
        }
    }
    if (fBegin > fEnd) {
        throw new BadRequestException("finishTimeEnd must be greater than finishTimeBegin");
    }
    Set<String> appTypes = parseQueries(applicationTypes, false);
    if (!appTypes.isEmpty()) {
        checkAppTypes = true;
    }
    // stateQuery is deprecated.
    if (stateQuery != null && !stateQuery.isEmpty()) {
        statesQuery.add(stateQuery);
    }
    Set<String> appStates = parseQueries(statesQuery, true);
    if (!appStates.isEmpty()) {
        checkAppStates = true;
    }
    AppsInfo allApps = new AppsInfo();
    Collection<ApplicationReport> appReports = null;
    final GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    request.setLimit(countNum);
    request.setStartRange(new LongRange(sBegin, sEnd));
    try {
        if (callerUGI == null) {
            // TODO: the request should take the params like what RMWebServices does
            // in YARN-1819.
            appReports = appBaseProt.getApplications(request).getApplicationList();
        } else {
            appReports = callerUGI.doAs(new PrivilegedExceptionAction<Collection<ApplicationReport>>() {

                @Override
                public Collection<ApplicationReport> run() throws Exception {
                    return appBaseProt.getApplications(request).getApplicationList();
                }
            });
        }
    } catch (Exception e) {
        rewrapAndThrowException(e);
    }
    if (appReports == null) {
        return allApps;
    }
    for (ApplicationReport appReport : appReports) {
        if (checkAppStates && !appStates.contains(StringUtils.toLowerCase(appReport.getYarnApplicationState().toString()))) {
            continue;
        }
        if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
            FinalApplicationStatus.valueOf(finalStatusQuery);
            if (!appReport.getFinalApplicationStatus().toString().equalsIgnoreCase(finalStatusQuery)) {
                continue;
            }
        }
        if (userQuery != null && !userQuery.isEmpty()) {
            if (!appReport.getUser().equals(userQuery)) {
                continue;
            }
        }
        if (queueQuery != null && !queueQuery.isEmpty()) {
            if (!appReport.getQueue().equals(queueQuery)) {
                continue;
            }
        }
        if (checkAppTypes && !appTypes.contains(StringUtils.toLowerCase(appReport.getApplicationType().trim()))) {
            continue;
        }
        if (checkEnd && (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) {
            continue;
        }
        AppInfo app = new AppInfo(appReport);
        allApps.add(app);
    }
    return allApps;
}

Example 18 with GetApplicationsRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest in project hadoop by apache.

the class AHSClientImpl method getApplications.

@Override
public List<ApplicationReport> getApplications() throws YarnException, IOException {
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(null, null);
    GetApplicationsResponse response = ahsClient.getApplications(request);
    return response.getApplicationList();
}

Example 19 with GetApplicationsRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest in project hadoop by apache.

the class TopCLI method fetchAppReports.

protected List<ApplicationReport> fetchAppReports() throws YarnException, IOException {
    List<ApplicationReport> ret;
    EnumSet<YarnApplicationState> states = EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING);
    GetApplicationsRequest req = GetApplicationsRequest.newInstance(types, states);
    req.setQueues(queues);
    req.setUsers(users);
    ret = applicationReportsCache.getIfPresent(req);
    if (ret != null) {
        return ret;
    }
    ret = client.getApplications(queues, users, types, states);
    applicationReportsCache.put(req, ret);
    return ret;
}

Example 20 with GetApplicationsRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest in project oozie by apache.

the class LauncherMainException method getChildYarnJobs.

public static Set<ApplicationId> getChildYarnJobs(Configuration actionConf, ApplicationsRequestScope scope, long startTime) {
    Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
    String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS);
    if (tag == null) {
        System.out.print("Could not find YARN tags property " + CHILD_MAPREDUCE_JOB_TAGS);
        return childYarnJobs;
    }
    System.out.println("tag id : " + tag);
    GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
    gar.setScope(scope);
    gar.setApplicationTags(Collections.singleton(tag));
    long endTime = System.currentTimeMillis();
    if (startTime > endTime) {
        System.out.println("WARNING: Clock skew between the Oozie server host and this host detected.  Please fix this.  " + "Attempting to work around...");
        // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an
        // offset in both directions
        long diff = 2 * (startTime - endTime);
        startTime = startTime - diff;
        endTime = endTime + diff;
    }
    gar.setStartRange(startTime, endTime);
    try {
        ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
        GetApplicationsResponse apps = proxy.getApplications(gar);
        List<ApplicationReport> appsList = apps.getApplicationList();
        for (ApplicationReport appReport : appsList) {
            childYarnJobs.add(appReport.getApplicationId());
        }
    } catch (YarnException | IOException ioe) {
        throw new RuntimeException("Exception occurred while finding child jobs", ioe);
    }
    if (childYarnJobs.isEmpty()) {
        System.out.println("No child applications found");
    } else {
        System.out.println("Found child YARN applications: " + StringUtils.join(childYarnJobs, ","));
    }
    return childYarnJobs;
}