Examples with RMAppAttemptContainerFinishedEvent org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent used on opensource projects

Example 1 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class RMAppAttemptImpl method rememberTargetTransitionsAndStoreState.

private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, Object transitionToDo, RMAppAttemptState targetFinalState, RMAppAttemptState stateToBeStored) {
    rememberTargetTransitions(event, transitionToDo, targetFinalState);
    stateBeforeFinalSaving = getState();
    // As of today, finalState, diagnostics, final-tracking-url and
    // finalAppStatus are the only things that we store into the StateStore
    // AFTER the initial saving on app-attempt-start
    // These fields can be visible from outside only after they are saved in
    // StateStore
    String diags = null;
    // don't leave the tracking URL pointing to a non-existent AM
    if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
        setTrackingUrlToAHSPage(stateToBeStored);
    } else {
        setTrackingUrlToRMAppPage(stateToBeStored);
    }
    String finalTrackingUrl = getOriginalTrackingUrl();
    FinalApplicationStatus finalStatus = null;
    int exitStatus = ContainerExitStatus.INVALID;
    switch(event.getType()) {
        case LAUNCH_FAILED:
            diags = event.getDiagnosticMsg();
            break;
        case REGISTERED:
            diags = getUnexpectedAMRegisteredDiagnostics();
            break;
        case UNREGISTERED:
            RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event;
            diags = unregisterEvent.getDiagnosticMsg();
            // reset finalTrackingUrl to url sent by am
            finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
            finalStatus = unregisterEvent.getFinalApplicationStatus();
            break;
        case CONTAINER_FINISHED:
            RMAppAttemptContainerFinishedEvent finishEvent = (RMAppAttemptContainerFinishedEvent) event;
            diags = getAMContainerCrashedDiagnostics(finishEvent);
            exitStatus = finishEvent.getContainerStatus().getExitStatus();
            break;
        case KILL:
            break;
        case FAIL:
            diags = event.getDiagnosticMsg();
            break;
        case EXPIRE:
            diags = getAMExpiredDiagnostics(event);
            break;
        default:
            break;
    }
    AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage();
    RMStateStore rmStore = rmContext.getStateStore();
    setFinishTime(System.currentTimeMillis());
    ApplicationAttemptStateData attemptState = ApplicationAttemptStateData.newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus, getFinishTime(), resUsage.getMemorySeconds(), resUsage.getVcoreSeconds(), this.attemptMetrics.getPreemptedMemory(), this.attemptMetrics.getPreemptedVcore());
    LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus);
    rmStore.updateApplicationAttemptState(attemptState);
}

Example 2 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testRunningToKilled.

@Test
public void testRunningToKilled() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL));
    // ignored ContainerFinished and Expire at FinalSaving if we were supposed
    // to Killed state.
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.KILLED, applicationAttempt.getAppAttemptState());
    assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
    assertEquals(amContainer, applicationAttempt.getMasterContainer());
    assertEquals(0, application.getRanNodes().size());
    String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app", applicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
    verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
    verifyAMHostAndPortInvalidated();
    verifyApplicationAttemptFinished(RMAppAttemptState.KILLED);
}

Example 3 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testContainersCleanupForLastAttempt.

@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
    // create a failed attempt.
    applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(submissionContext.getKeepContainersAcrossApplicationAttempts()).thenReturn(true);
    when(submissionContext.getMaxAppAttempts()).thenReturn(1);
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123);
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs1, anyNodeId));
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    assertFalse(transferStateFromPreviousAttempt);
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}

Example 4 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMContainerImpl method testReleaseWhileRunning.

@Test
public void testReleaseWhileRunning() {
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
    EventHandler generic = mock(EventHandler.class);
    drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler);
    drainDispatcher.register(RMNodeEventType.class, generic);
    drainDispatcher.init(new YarnConfiguration());
    drainDispatcher.start();
    NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    Resource resource = BuilderUtils.newResource(512, 1);
    Priority priority = BuilderUtils.newPriority(5);
    Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null);
    ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(null);
    Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId) Matchers.any());
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getRMApps()).thenReturn(rmApps);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO, true);
    when(rmContext.getYarnConfiguration()).thenReturn(conf);
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, nodeId, "user", rmContext);
    assertEquals(RMContainerState.NEW, rmContainer.getState());
    assertEquals(resource, rmContainer.getAllocatedResource());
    assertEquals(nodeId, rmContainer.getAllocatedNode());
    assertEquals(priority, rmContainer.getAllocatedSchedulerKey().getPriority());
    verify(writer).containerStarted(any(RMContainer.class));
    verify(publisher).containerCreated(any(RMContainer.class), anyLong());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
    drainDispatcher.await();
    assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED));
    drainDispatcher.await();
    assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
    drainDispatcher.await();
    assertEquals(RMContainerState.RUNNING, rmContainer.getState());
    assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL());
    // In RUNNING state. Verify RELEASED and associated actions.
    reset(appAttemptEventHandler);
    ContainerStatus containerStatus = SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER);
    rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED));
    drainDispatcher.await();
    assertEquals(RMContainerState.RELEASED, rmContainer.getState());
    assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo());
    assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus());
    assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
    verify(writer).containerFinished(any(RMContainer.class));
    verify(publisher).containerFinished(any(RMContainer.class), anyLong());
    ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.forClass(RMAppAttemptContainerFinishedEvent.class);
    verify(appAttemptEventHandler).handle(captor.capture());
    RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
    assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
    assertEquals(containerStatus, cfEvent.getContainerStatus());
    assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
    // In RELEASED state. A FINIHSED event may come in.
    rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED));
    assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}

Example 5 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testUsageReport.

@Test
public void testUsageReport() {
    // scheduler has info on running apps
    ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
    ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class);
    when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
    when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
    // start and finish the attempt
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId, "", FinalApplicationStatus.SUCCEEDED, ""));
    // expect usage stats to come from the scheduler report
    ApplicationResourceUsageReport report = applicationAttempt.getApplicationResourceUsageReport();
    Assert.assertEquals(123456L, report.getMemorySeconds());
    Assert.assertEquals(55544L, report.getVcoreSeconds());
    // finish app attempt and remove it from scheduler 
    when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
    sendAttemptUpdateSavedEvent(applicationAttempt);
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(attemptId, ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
    when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
    report = applicationAttempt.getApplicationResourceUsageReport();
    Assert.assertEquals(223456, report.getMemorySeconds());
    Assert.assertEquals(75544, report.getVcoreSeconds());
}