Examples with ContainerLaunchContext org.apache.hadoop.yarn.api.records.ContainerLaunchContext used on opensource projects

Example 71 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class Application method submit.

@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
    ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    context.setApplicationId(this.applicationId);
    context.setQueue(this.queue);
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    context.setAMContainerSpec(amContainer);
    context.setResource(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    SubmitApplicationRequest request = recordFactory.newRecordInstance(SubmitApplicationRequest.class);
    request.setApplicationSubmissionContext(context);
    final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    resourceManager.getClientRMService().submitApplication(request);
    RMAppEvent event = new RMAppEvent(this.applicationId, RMAppEventType.START);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    event = new RMAppEvent(this.applicationId, RMAppEventType.APP_NEW_SAVED);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    event = new RMAppEvent(this.applicationId, RMAppEventType.APP_ACCEPTED);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    // Notify scheduler
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
    scheduler.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
    scheduler.handle(addAttemptEvent);
}

Example 72 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestApplicationACLs method submitAppAndGetAppId.

@SuppressWarnings("deprecation")
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) throws Exception {
    SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);
    ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    ApplicationId applicationId = rmClient.getNewApplication(recordFactory.newRecordInstance(GetNewApplicationRequest.class)).getApplicationId();
    context.setApplicationId(applicationId);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>();
    acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
    acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());
    ContainerLaunchContext amContainer = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    Resource resource = BuilderUtils.newResource(1024, 1);
    context.setResource(resource);
    amContainer.setApplicationACLs(acls);
    context.setAMContainerSpec(amContainer);
    submitRequest.setApplicationSubmissionContext(context);
    rmClient.submitApplication(submitRequest);
    resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
    return applicationId;
}

Example 73 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestApplicationACLs method verifyInvalidQueueWithAcl.

private void verifyInvalidQueueWithAcl() throws Exception {
    isQueueUser = true;
    SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);
    ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    ApplicationId applicationId = rmClient.getNewApplication(recordFactory.newRecordInstance(GetNewApplicationRequest.class)).getApplicationId();
    context.setApplicationId(applicationId);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>();
    ContainerLaunchContext amContainer = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    Resource resource = BuilderUtils.newResource(1024, 1);
    context.setResource(resource);
    amContainer.setApplicationACLs(acls);
    context.setQueue("InvalidQueue");
    context.setAMContainerSpec(amContainer);
    submitRequest.setApplicationSubmissionContext(context);
    rmClient.submitApplication(submitRequest);
    resourceManager.waitForState(applicationId, RMAppState.FAILED);
    final GetApplicationReportRequest appReportRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    GetApplicationReportResponse applicationReport = rmClient.getApplicationReport(appReportRequest);
    ApplicationReport appReport = applicationReport.getApplicationReport();
    Assert.assertTrue(appReport.getDiagnostics().contains("submitted by user owner to unknown queue: InvalidQueue"));
}

Example 74 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishApplicationMetrics.

@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
    long stateUpdateTimeStamp = System.currentTimeMillis();
    for (int i = 1; i <= 2; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        RMApp app = createRMApp(appId);
        metricsPublisher.appCreated(app, app.getStartTime());
        if (i == 1) {
            when(app.getQueue()).thenReturn("new test queue");
            ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
            when(asc.getUnmanagedAM()).thenReturn(false);
            when(asc.getPriority()).thenReturn(Priority.newInstance(1));
            when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
            ContainerLaunchContext containerLaunchContext = mock(ContainerLaunchContext.class);
            when(containerLaunchContext.getCommands()).thenReturn(Collections.singletonList("java -Xmx1024m"));
            when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
            when(app.getApplicationSubmissionContext()).thenReturn(asc);
            when(app.getApplicationPriority()).thenReturn(Priority.newInstance(1));
            metricsPublisher.appUpdated(app, 4L);
        } else {
            metricsPublisher.appUpdated(app, 4L);
        }
        metricsPublisher.appStateUpdated(app, YarnApplicationState.RUNNING, stateUpdateTimeStamp);
        metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
        if (i == 1) {
            metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
        } else {
            // in case user doesn't specify the ACLs
            metricsPublisher.appACLsUpdated(app, null, 4L);
        }
        TimelineEntity entity = null;
        do {
            entity = store.getEntity(appId.toString(), ApplicationMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
        // ensure Five events are both published before leaving the loop
        } while (entity == null || entity.getEvents().size() < 5);
        // verify all the fields
        Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE, entity.getEntityType());
        Assert.assertEquals(app.getApplicationId().toString(), entity.getEntityId());
        Assert.assertEquals(app.getName(), entity.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENTITY_INFO));
        if (i != 1) {
            Assert.assertEquals(app.getQueue(), entity.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
        }
        Assert.assertEquals(app.getApplicationSubmissionContext().getUnmanagedAM(), entity.getOtherInfo().get(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
        if (i != 1) {
            Assert.assertEquals(app.getApplicationSubmissionContext().getPriority().getPriority(), entity.getOtherInfo().get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
        }
        Assert.assertEquals(app.getAmNodeLabelExpression(), entity.getOtherInfo().get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION));
        Assert.assertEquals(app.getApplicationSubmissionContext().getNodeLabelExpression(), entity.getOtherInfo().get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION));
        Assert.assertEquals(app.getUser(), entity.getOtherInfo().get(ApplicationMetricsConstants.USER_ENTITY_INFO));
        Assert.assertEquals(app.getApplicationType(), entity.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENTITY_INFO));
        Assert.assertEquals(app.getSubmitTime(), entity.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
        Assert.assertTrue(verifyAppTags(app.getApplicationTags(), entity.getOtherInfo()));
        if (i == 1) {
            Assert.assertEquals("uers1,user2", entity.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
            Assert.assertEquals(app.getApplicationSubmissionContext().getAMContainerSpec().getCommands(), entity.getOtherInfo().get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND));
        } else {
            Assert.assertEquals("", entity.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
            Assert.assertEquals(app.getRMAppMetrics().getMemorySeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getPreemptedMemorySeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getPreemptedVcoreSeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS).toString()));
        }
        Assert.assertEquals("context", entity.getOtherInfo().get(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT));
        boolean hasCreatedEvent = false;
        boolean hasUpdatedEvent = false;
        boolean hasFinishedEvent = false;
        boolean hasACLsUpdatedEvent = false;
        boolean hasStateUpdateEvent = false;
        for (TimelineEvent event : entity.getEvents()) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                hasCreatedEvent = true;
                Assert.assertEquals(app.getStartTime(), event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                hasFinishedEvent = true;
                Assert.assertEquals(app.getFinishTime(), event.getTimestamp());
                Assert.assertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
                Assert.assertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
                Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            } else if (event.getEventType().equals(ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
                hasUpdatedEvent = true;
                Assert.assertEquals(4L, event.getTimestamp());
                if (1 == i) {
                    Assert.assertEquals(1, event.getEventInfo().get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
                    Assert.assertEquals("new test queue", event.getEventInfo().get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
                }
            } else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
                hasACLsUpdatedEvent = true;
                Assert.assertEquals(4L, event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) {
                hasStateUpdateEvent = true;
                Assert.assertEquals(event.getTimestamp(), stateUpdateTimeStamp);
                Assert.assertEquals(YarnApplicationState.RUNNING.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            }
        }
        // Do assertTrue verification separately for easier debug
        Assert.assertTrue(hasCreatedEvent);
        Assert.assertTrue(hasFinishedEvent);
        Assert.assertTrue(hasACLsUpdatedEvent);
        Assert.assertTrue(hasUpdatedEvent);
        Assert.assertTrue(hasStateUpdateEvent);
    }
}

Example 75 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestRMAppTransitions method testAppRecoverPath.

@Test(timeout = 30000)
public void testAppRecoverPath() throws IOException {
    LOG.info("--- START: testAppRecoverPath ---");
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class);
    Credentials credentials = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    clc.setTokens(securityTokens);
    sub.setAMContainerSpec(clc);
    testCreateAppSubmittedRecovery(sub);
}