Examples with GetApplicationsResponse org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse used on opensource projects

Example 11 with GetApplicationsResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse in project hive by apache.

the class WebHCatJTShim23 method getYarnChildJobs.

/**
 * Queries RM for the list of applications with the given tag that have started
 * after the given timestamp.
 */
private Set<ApplicationId> getYarnChildJobs(String tag, long timestamp) {
    Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
    LOG.info(String.format("Querying RM for tag = %s, starting with ts = %s", tag, timestamp));
    GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
    gar.setScope(ApplicationsRequestScope.OWN);
    gar.setStartRange(timestamp, System.currentTimeMillis());
    gar.setApplicationTags(Collections.singleton(tag));
    try {
        ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
        GetApplicationsResponse apps = proxy.getApplications(gar);
        List<ApplicationReport> appsList = apps.getApplicationList();
        for (ApplicationReport appReport : appsList) {
            childYarnJobs.add(appReport.getApplicationId());
        }
    } catch (IOException ioe) {
        throw new RuntimeException("Exception occurred while finding child jobs", ioe);
    } catch (YarnException ye) {
        throw new RuntimeException("Exception occurred while finding child jobs", ye);
    }
    return childYarnJobs;
}

Example 12 with GetApplicationsResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse in project hadoop by apache.

the class TestRMWebApp method mockClientRMService.

public static ClientRMService mockClientRMService(RMContext rmContext) {
    ClientRMService clientRMService = mock(ClientRMService.class);
    List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
    for (RMApp app : rmContext.getRMApps().values()) {
        ApplicationReport appReport = ApplicationReport.newInstance(app.getApplicationId(), (ApplicationAttemptId) null, app.getUser(), app.getQueue(), app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null);
        appReports.add(appReport);
    }
    GetApplicationsResponse response = mock(GetApplicationsResponse.class);
    when(response.getApplicationList()).thenReturn(appReports);
    try {
        when(clientRMService.getApplications(any(GetApplicationsRequest.class))).thenReturn(response);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    return clientRMService;
}

Example 13 with GetApplicationsResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse in project hadoop by apache.

the class TestRMRestart method testRMRestartGetApplicationList.

@Test(timeout = 60000)
public void testRMRestartGetApplicationList() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm1);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // a succeeded app.
    RMApp app0 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am0 = launchAM(app0, rm1, nm1);
    finishApplicationMaster(app0, rm1, nm1, am0);
    // a failed app.
    RMApp app1 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am1 = launchAM(app1, rm1, nm1);
    // fail the AM by sending CONTAINER_FINISHED event without registering.
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED);
    // a killed app.
    RMApp app2 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am2 = launchAM(app2, rm1, nm1);
    rm1.killApp(app2.getApplicationId());
    rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    verify(rm1.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    // restart rm
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        protected RMAppManager createRMAppManager() {
            return spy(super.createRMAppManager());
        }

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm2);
    rm2.start();
    verify(rm2.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    GetApplicationsRequest request1 = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED));
    GetApplicationsResponse response1 = rm2.getClientRMService().getApplications(request1);
    List<ApplicationReport> appList1 = response1.getApplicationList();
    // assert all applications exist according to application state after RM
    // restarts.
    boolean forApp0 = false, forApp1 = false, forApp2 = false;
    for (ApplicationReport report : appList1) {
        if (report.getApplicationId().equals(app0.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FINISHED, report.getYarnApplicationState());
            forApp0 = true;
        }
        if (report.getApplicationId().equals(app1.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FAILED, report.getYarnApplicationState());
            forApp1 = true;
        }
        if (report.getApplicationId().equals(app2.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.KILLED, report.getYarnApplicationState());
            forApp2 = true;
        }
    }
    Assert.assertTrue(forApp0 && forApp1 && forApp2);
    // assert all applications exist according to application type after RM
    // restarts.
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("myType");
    GetApplicationsRequest request2 = GetApplicationsRequest.newInstance(appTypes);
    GetApplicationsResponse response2 = rm2.getClientRMService().getApplications(request2);
    List<ApplicationReport> appList2 = response2.getApplicationList();
    Assert.assertTrue(3 == appList2.size());
    // check application summary is logged for the completed apps with timeout
    // to make sure APP_COMPLETED events are processed, after RM restart.
    verify(rm2.getRMAppManager(), timeout(1000).times(3)).logApplicationSummary(isA(ApplicationId.class));
}

Example 14 with GetApplicationsResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse in project hadoop by apache.

the class ApplicationHistoryClientService method getApplications.

@Override
public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException {
    long startedBegin = request.getStartRange() == null ? 0L : request.getStartRange().getMinimumLong();
    long startedEnd = request.getStartRange() == null ? Long.MAX_VALUE : request.getStartRange().getMaximumLong();
    GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(history.getApplications(request.getLimit(), startedBegin, startedEnd).values()));
    return response;
}

Example 15 with GetApplicationsResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse in project hadoop by apache.

the class ClientRMService method getApplications.

/**
   * Get applications matching the {@link GetApplicationsRequest}. If
   * caseSensitive is set to false, applicationTypes in
   * GetApplicationRequest are expected to be in all-lowercase
   */
@Private
public GetApplicationsResponse getApplications(GetApplicationsRequest request, boolean caseSensitive) throws YarnException {
    UserGroupInformation callerUGI;
    try {
        callerUGI = UserGroupInformation.getCurrentUser();
    } catch (IOException ie) {
        LOG.info("Error getting UGI ", ie);
        throw RPCUtil.getRemoteException(ie);
    }
    Set<String> applicationTypes = request.getApplicationTypes();
    EnumSet<YarnApplicationState> applicationStates = request.getApplicationStates();
    Set<String> users = request.getUsers();
    Set<String> queues = request.getQueues();
    Set<String> tags = request.getApplicationTags();
    long limit = request.getLimit();
    LongRange start = request.getStartRange();
    LongRange finish = request.getFinishRange();
    ApplicationsRequestScope scope = request.getScope();
    final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
    Iterator<RMApp> appsIter;
    // of those queues by asking the scheduler for the apps in those queues.
    if (queues != null && !queues.isEmpty()) {
        // Construct an iterator over apps in given queues
        // Collect list of lists to avoid copying all apps
        final List<List<ApplicationAttemptId>> queueAppLists = new ArrayList<List<ApplicationAttemptId>>();
        for (String queue : queues) {
            List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue);
            if (appsInQueue != null && !appsInQueue.isEmpty()) {
                queueAppLists.add(appsInQueue);
            }
        }
        appsIter = new Iterator<RMApp>() {

            Iterator<List<ApplicationAttemptId>> appListIter = queueAppLists.iterator();

            Iterator<ApplicationAttemptId> schedAppsIter;

            @Override
            public boolean hasNext() {
                // current list hasNext or whether there are any remaining lists
                return (schedAppsIter != null && schedAppsIter.hasNext()) || appListIter.hasNext();
            }

            @Override
            public RMApp next() {
                if (schedAppsIter == null || !schedAppsIter.hasNext()) {
                    schedAppsIter = appListIter.next().iterator();
                }
                return apps.get(schedAppsIter.next().getApplicationId());
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException("Remove not supported");
            }
        };
    } else {
        appsIter = apps.values().iterator();
    }
    List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
    while (appsIter.hasNext() && reports.size() < limit) {
        RMApp application = appsIter.next();
        // Check if current application falls under the specified scope
        if (scope == ApplicationsRequestScope.OWN && !callerUGI.getUserName().equals(application.getUser())) {
            continue;
        }
        if (applicationTypes != null && !applicationTypes.isEmpty()) {
            String appTypeToMatch = caseSensitive ? application.getApplicationType() : StringUtils.toLowerCase(application.getApplicationType());
            if (!applicationTypes.contains(appTypeToMatch)) {
                continue;
            }
        }
        if (applicationStates != null && !applicationStates.isEmpty()) {
            if (!applicationStates.contains(application.createApplicationState())) {
                continue;
            }
        }
        if (users != null && !users.isEmpty() && !users.contains(application.getUser())) {
            continue;
        }
        if (start != null && !start.containsLong(application.getStartTime())) {
            continue;
        }
        if (finish != null && !finish.containsLong(application.getFinishTime())) {
            continue;
        }
        if (tags != null && !tags.isEmpty()) {
            Set<String> appTags = application.getApplicationTags();
            if (appTags == null || appTags.isEmpty()) {
                continue;
            }
            boolean match = false;
            for (String tag : tags) {
                if (appTags.contains(tag)) {
                    match = true;
                    break;
                }
            }
            if (!match) {
                continue;
            }
        }
        // checkAccess can grab the scheduler lock so call it last
        boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application);
        if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) {
            continue;
        }
        reports.add(application.createAndGetApplicationReport(callerUGI.getUserName(), allowAccess));
    }
    GetApplicationsResponse response = recordFactory.newRecordInstance(GetApplicationsResponse.class);
    response.setApplicationList(reports);
    return response;
}