Examples with SubmitApplicationRequest org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest used on opensource projects

Example 6 with SubmitApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest in project hadoop by apache.

the class MockRM method submitApp.

public RMApp submitApp(Resource capability, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval, LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete, Priority priority, String amLabel, Map<ApplicationTimeoutType, Long> applicationTimeouts, ByteBuffer tokensConf) throws Exception {
    ApplicationId appId = isAppIdProvided ? applicationId : null;
    ApplicationClientProtocol client = getClientRMService();
    if (!isAppIdProvided) {
        GetNewApplicationResponse resp = client.getNewApplication(Records.newRecord(GetNewApplicationRequest.class));
        appId = resp.getApplicationId();
    }
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setKeepContainersAcrossApplicationAttempts(keepContainers);
    sub.setApplicationId(appId);
    sub.setApplicationName(name);
    sub.setMaxAppAttempts(maxAppAttempts);
    if (applicationTimeouts != null && applicationTimeouts.size() > 0) {
        sub.setApplicationTimeouts(applicationTimeouts);
    }
    if (unmanaged) {
        sub.setUnmanagedAM(true);
    }
    if (queue != null) {
        sub.setQueue(queue);
    }
    if (priority != null) {
        sub.setPriority(priority);
    }
    sub.setApplicationType(appType);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class);
    sub.setResource(capability);
    clc.setApplicationACLs(acls);
    if (ts != null && UserGroupInformation.isSecurityEnabled()) {
        DataOutputBuffer dob = new DataOutputBuffer();
        ts.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        clc.setTokens(securityTokens);
        clc.setTokensConf(tokensConf);
    }
    sub.setAMContainerSpec(clc);
    sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    if (logAggregationContext != null) {
        sub.setLogAggregationContext(logAggregationContext);
    }
    sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
    ResourceRequest amResourceRequest = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
    if (amLabel != null && !amLabel.isEmpty()) {
        amResourceRequest.setNodeLabelExpression(amLabel.trim());
    }
    sub.setAMContainerResourceRequest(amResourceRequest);
    req.setApplicationSubmissionContext(sub);
    UserGroupInformation fakeUser = UserGroupInformation.createUserForTesting(user, new String[] { "someGroup" });
    PrivilegedExceptionAction<SubmitApplicationResponse> action = new PrivilegedExceptionAction<SubmitApplicationResponse>() {

        ApplicationClientProtocol client;

        SubmitApplicationRequest req;

        @Override
        public SubmitApplicationResponse run() throws IOException, YarnException {
            try {
                return client.submitApplication(req);
            } catch (YarnException | IOException e) {
                e.printStackTrace();
                throw e;
            }
        }

        PrivilegedExceptionAction<SubmitApplicationResponse> setClientReq(ApplicationClientProtocol client, SubmitApplicationRequest req) {
            this.client = client;
            this.req = req;
            return this;
        }
    }.setClientReq(client, req);
    fakeUser.doAs(action);
    // make sure app is immediately available after submit
    if (waitForAccepted) {
        waitForState(appId, RMAppState.ACCEPTED);
    }
    RMApp rmApp = getRMContext().getRMApps().get(appId);
    // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
    if (waitForAccepted && !unmanaged) {
        waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.SCHEDULED);
    }
    return rmApp;
}

Example 7 with SubmitApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest in project hadoop by apache.

the class TestClientRMService method testGetApplications.

@Test
public void testGetApplications() throws IOException, YarnException {
    /**
     * 1. Submit 3 applications alternately in two queues
     * 2. Test each of the filters
     */
    // Basic setup
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // Initialize appnames and queues
    String[] queues = { QUEUE_1, QUEUE_2 };
    String[] appNames = { MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName() };
    ApplicationId[] appIds = { getApplicationId(101), getApplicationId(102), getApplicationId(103) };
    List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
    long[] submitTimeMillis = new long[3];
    // Submit applications
    for (int i = 0; i < appIds.length; i++) {
        ApplicationId appId = appIds[i];
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true);
        SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, appNames[i], queues[i % queues.length], new HashSet<String>(tags.subList(0, i + 1)));
        rmService.submitApplication(submitRequest);
        submitTimeMillis[i] = System.currentTimeMillis();
    }
    // Test different cases of ClientRMService#getApplications()
    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    assertEquals("Incorrect total number of apps", 6, rmService.getApplications(request).getApplicationList().size());
    // Check limit
    request.setLimit(1L);
    assertEquals("Failed to limit applications", 1, rmService.getApplications(request).getApplicationList().size());
    // Check start range
    request = GetApplicationsRequest.newInstance();
    request.setStartRange(submitTimeMillis[0], System.currentTimeMillis());
    // 2 applications are submitted after first timeMills
    assertEquals("Incorrect number of matching start range", 2, rmService.getApplications(request).getApplicationList().size());
    // 1 application is submitted after the second timeMills
    request.setStartRange(submitTimeMillis[1], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 1, rmService.getApplications(request).getApplicationList().size());
    // no application is submitted after the third timeMills
    request.setStartRange(submitTimeMillis[2], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 0, rmService.getApplications(request).getApplicationList().size());
    // Check queue
    request = GetApplicationsRequest.newInstance();
    Set<String> queueSet = new HashSet<String>();
    request.setQueues(queueSet);
    queueSet.add(queues[0]);
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request).getApplicationList().size());
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request, false).getApplicationList().size());
    queueSet.add(queues[1]);
    assertEquals("Incorrect number of applications in queue", 3, rmService.getApplications(request).getApplicationList().size());
    // Check user
    request = GetApplicationsRequest.newInstance();
    Set<String> userSet = new HashSet<String>();
    request.setUsers(userSet);
    userSet.add("random-user-name");
    assertEquals("Incorrect number of applications for user", 0, rmService.getApplications(request).getApplicationList().size());
    userSet.add(UserGroupInformation.getCurrentUser().getShortUserName());
    assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size());
    // Check tags
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.ALL, null, null, null, null, null, null, null, null);
    Set<String> tagSet = new HashSet<String>();
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 6, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(0));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 3, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(1));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 2, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(2));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 1, rmService.getApplications(request).getApplicationList().size());
    // Check scope
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.VIEWABLE);
    assertEquals("Incorrect number of applications for the scope", 6, rmService.getApplications(request).getApplicationList().size());
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.OWN);
    assertEquals("Incorrect number of applications for the scope", 3, rmService.getApplications(request).getApplicationList().size());
}

Example 8 with SubmitApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest in project hadoop by apache.

the class TestClientRMService method testAppSubmit.

@Test(timeout = 30000)
@SuppressWarnings("rawtypes")
public void testAppSubmit() throws Exception {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    ApplicationId appId1 = getApplicationId(100);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // without name and queue
    SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    try {
        rmService.submitApplication(submitRequest1);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app1 = rmContext.getRMApps().get(appId1);
    Assert.assertNotNull("app doesn't exist", app1);
    Assert.assertEquals("app name doesn't match", YarnConfiguration.DEFAULT_APPLICATION_NAME, app1.getName());
    Assert.assertEquals("app queue doesn't match", YarnConfiguration.DEFAULT_QUEUE_NAME, app1.getQueue());
    // with name and queue
    String name = MockApps.newAppName();
    String queue = MockApps.newQueue();
    ApplicationId appId2 = getApplicationId(101);
    SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, name, queue);
    submitRequest2.getApplicationSubmissionContext().setApplicationType("matchType");
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app2 = rmContext.getRMApps().get(appId2);
    Assert.assertNotNull("app doesn't exist", app2);
    Assert.assertEquals("app name doesn't match", name, app2.getName());
    Assert.assertEquals("app queue doesn't match", queue, app2.getQueue());
    // duplicate appId
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    GetApplicationsRequest getAllAppsRequest = GetApplicationsRequest.newInstance(new HashSet<String>());
    GetApplicationsResponse getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(5, getAllApplicationsResponse.getApplicationList().size());
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("matchType");
    getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes);
    getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(1, getAllApplicationsResponse.getApplicationList().size());
    Assert.assertEquals(appId2, getAllApplicationsResponse.getApplicationList().get(0).getApplicationId());
}

Example 9 with SubmitApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest in project hadoop by apache.

the class TestAppManager method testQueueSubmitWithNoPermission.

@Test
public void testQueueSubmitWithNoPermission() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName());
    conf.set(PREFIX + "root.acl_submit_applications", " ");
    conf.set(PREFIX + "root.acl_administer_queue", " ");
    conf.set(PREFIX + "root.default.acl_submit_applications", " ");
    conf.set(PREFIX + "root.default.acl_administer_queue", " ");
    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
    MockRM mockRM = new MockRM(conf);
    ClientRMService rmService = mockRM.getClientRMService();
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setApplicationId(appId);
    ResourceRequest resReg = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
    sub.setAMContainerResourceRequest(resReg);
    req.setApplicationSubmissionContext(sub);
    sub.setAMContainerSpec(mock(ContainerLaunchContext.class));
    try {
        rmService.submitApplication(req);
    } catch (Exception e) {
        e.printStackTrace();
        if (e instanceof YarnException) {
            Assert.assertTrue(e.getCause() instanceof AccessControlException);
        } else {
            Assert.fail("Yarn exception is expected : " + e.getMessage());
        }
    } finally {
        mockRM.close();
    }
}

Example 10 with SubmitApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest in project hadoop by apache.

the class TestDelegationTokenRenewer method testAppSubmissionWithInvalidDelegationToken.

@Test(timeout = 20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    MockRM rm = new MockRM(conf) {

        @Override
        protected void doSecureLogin() throws IOException {
        // Skip the login.
        }
    };
    ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes());
    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>());
    ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance(ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1), "BOGUS");
    SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext);
    try {
        rm.getClientRMService().submitApplication(request);
        fail("Error was excepted.");
    } catch (YarnException e) {
        Assert.assertTrue(e.getMessage().contains("Bad header found in token storage"));
    }
}