Examples with ApplicationSubmissionContext org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext used on opensource projects

Example 6 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class QueueACLsTestBase method submitAppAndGetAppId.

private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception {
    GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance();
    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId();
    Resource resource = BuilderUtils.newResource(1024, 1);
    Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
    ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls);
    ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType");
    appSubmissionContext.setApplicationId(applicationId);
    appSubmissionContext.setQueue(queueName);
    SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext);
    submitterClient.submitApplication(submitRequest);
    resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
    return applicationId;
}

Example 7 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class MockRM method submitApp.

public RMApp submitApp(Resource capability, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval, LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete, Priority priority, String amLabel, Map<ApplicationTimeoutType, Long> applicationTimeouts, ByteBuffer tokensConf) throws Exception {
    ApplicationId appId = isAppIdProvided ? applicationId : null;
    ApplicationClientProtocol client = getClientRMService();
    if (!isAppIdProvided) {
        GetNewApplicationResponse resp = client.getNewApplication(Records.newRecord(GetNewApplicationRequest.class));
        appId = resp.getApplicationId();
    }
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setKeepContainersAcrossApplicationAttempts(keepContainers);
    sub.setApplicationId(appId);
    sub.setApplicationName(name);
    sub.setMaxAppAttempts(maxAppAttempts);
    if (applicationTimeouts != null && applicationTimeouts.size() > 0) {
        sub.setApplicationTimeouts(applicationTimeouts);
    }
    if (unmanaged) {
        sub.setUnmanagedAM(true);
    }
    if (queue != null) {
        sub.setQueue(queue);
    }
    if (priority != null) {
        sub.setPriority(priority);
    }
    sub.setApplicationType(appType);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class);
    sub.setResource(capability);
    clc.setApplicationACLs(acls);
    if (ts != null && UserGroupInformation.isSecurityEnabled()) {
        DataOutputBuffer dob = new DataOutputBuffer();
        ts.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        clc.setTokens(securityTokens);
        clc.setTokensConf(tokensConf);
    }
    sub.setAMContainerSpec(clc);
    sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    if (logAggregationContext != null) {
        sub.setLogAggregationContext(logAggregationContext);
    }
    sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
    ResourceRequest amResourceRequest = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
    if (amLabel != null && !amLabel.isEmpty()) {
        amResourceRequest.setNodeLabelExpression(amLabel.trim());
    }
    sub.setAMContainerResourceRequest(amResourceRequest);
    req.setApplicationSubmissionContext(sub);
    UserGroupInformation fakeUser = UserGroupInformation.createUserForTesting(user, new String[] { "someGroup" });
    PrivilegedExceptionAction<SubmitApplicationResponse> action = new PrivilegedExceptionAction<SubmitApplicationResponse>() {

        ApplicationClientProtocol client;

        SubmitApplicationRequest req;

        @Override
        public SubmitApplicationResponse run() throws IOException, YarnException {
            try {
                return client.submitApplication(req);
            } catch (YarnException | IOException e) {
                e.printStackTrace();
                throw e;
            }
        }

        PrivilegedExceptionAction<SubmitApplicationResponse> setClientReq(ApplicationClientProtocol client, SubmitApplicationRequest req) {
            this.client = client;
            this.req = req;
            return this;
        }
    }.setClientReq(client, req);
    fakeUser.doAs(action);
    // make sure app is immediately available after submit
    if (waitForAccepted) {
        waitForState(appId, RMAppState.ACCEPTED);
    }
    RMApp rmApp = getRMContext().getRMApps().get(appId);
    // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
    if (waitForAccepted && !unmanaged) {
        waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.SCHEDULED);
    }
    return rmApp;
}

Example 8 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class TestYARNRunner method testAMProfiler.

@Test(timeout = 20000)
public void testAMProfiler() throws Exception {
    JobConf jobConf = new JobConf();
    jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);
    YARNRunner yarnRunner = new YARNRunner(jobConf);
    ApplicationSubmissionContext submissionContext = buildSubmitContext(yarnRunner, jobConf);
    ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
    List<String> commands = containerSpec.getCommands();
    for (String command : commands) {
        if (command != null) {
            if (command.contains(PROFILE_PARAMS)) {
                return;
            }
        }
    }
    throw new IllegalStateException("Profiler opts not found!");
}

Example 9 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class TestYARNRunner method setUp.

@Before
public void setUp() throws Exception {
    resourceMgrDelegate = mock(ResourceMgrDelegate.class);
    conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
    clientCache = new ClientCache(conf, resourceMgrDelegate);
    clientCache = spy(clientCache);
    yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
    yarnRunner = spy(yarnRunner);
    submissionContext = mock(ApplicationSubmissionContext.class);
    doAnswer(new Answer<ApplicationSubmissionContext>() {

        @Override
        public ApplicationSubmissionContext answer(InvocationOnMock invocation) throws Throwable {
            return submissionContext;
        }
    }).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class), any(String.class), any(Credentials.class));
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    jobId = TypeConverter.fromYarn(appId);
    if (testWorkDir.exists()) {
        FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true);
    }
    testWorkDir.mkdirs();
}

Example 10 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class AMLauncher method launch.

private void launch() throws IOException, YarnException {
    connect();
    ContainerId masterContainerID = masterContainer.getId();
    ApplicationSubmissionContext applicationContext = application.getSubmissionContext();
    LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId());
    ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerMgrProxy.startContainers(allRequests);
    if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) {
        Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize();
        parseAndThrowException(t);
    } else {
        LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId());
    }
}