Examples with RMAppAttemptContainerFinishedEvent org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent used on opensource projects

Example 6 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinalSavingToFinishedWithContainerFinished.

// While attempt is at FINAL_SAVING, Contaienr_Finished event may come before
// Attempt_Saved event, we stay on FINAL_SAVING on Container_Finished event
// and then directly jump from FINAL_SAVING to FINISHED state on Attempt_Saved
// event
@Test
public void testFinalSavingToFinishedWithContainerFinished() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
    String trackingUrl = "mytrackingurl";
    String diagnostics = "Successful";
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, diagnostics));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    // Container_finished event comes before Attempt_Saved event.
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    // send attempt_saved
    sendAttemptUpdateSavedEvent(applicationAttempt);
    testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, diagnostics, 0, false);
}

Example 7 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testLaunchedFailWhileAHSEnabled.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testLaunchedFailWhileAHSEnabled() {
    Configuration myConf = new Configuration(conf);
    myConf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
    ApplicationId applicationId = MockApps.newAppID(appId);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 2);
    RMAppAttempt myApplicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, myConf, BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    //submit, schedule and allocate app attempt
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.START));
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.ATTEMPT_ADDED));
    Container amContainer = mock(Container.class);
    Resource resource = BuilderUtils.newResource(2048, 1);
    when(amContainer.getId()).thenReturn(BuilderUtils.newContainerId(myApplicationAttempt.getAppAttemptId(), 1));
    when(amContainer.getResource()).thenReturn(resource);
    Allocation allocation = mock(Allocation.class);
    when(allocation.getContainers()).thenReturn(Collections.singletonList(amContainer));
    when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class))).thenReturn(allocation);
    RMContainer rmContainer = mock(RMContainerImpl.class);
    when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer);
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.CONTAINER_ALLOCATED));
    assertEquals(RMAppAttemptState.ALLOCATED_SAVING, myApplicationAttempt.getAppAttemptState());
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
    // launch app attempt
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.LAUNCHED));
    assertEquals(YarnApplicationAttemptState.LAUNCHED, myApplicationAttempt.createApplicationAttemptState());
    //fail container right after launched
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    myApplicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(myApplicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    sendAttemptUpdateSavedEvent(myApplicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, myApplicationAttempt.getAppAttemptState());
    String rmAppPageUrl = pjoin(AHS_WEBAPP_ADDR, "applicationhistory", "app", myApplicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, myApplicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, myApplicationAttempt.getTrackingUrl());
}

Example 8 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFailedToFailed.

@Test
public void testFailedToFailed() {
    // create a failed attempt.
    when(submissionContext.getKeepContainersAcrossApplicationAttempts()).thenReturn(true);
    when(application.getMaxAppAttempts()).thenReturn(2);
    when(application.getNumFailedAppAttempts()).thenReturn(1);
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123);
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs1, anyNodeId));
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    // should not kill containers when attempt fails.
    assertTrue(transferStateFromPreviousAttempt);
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
    // failed attempt captured the container finished event.
    assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
    ContainerStatus cs2 = ContainerStatus.newInstance(ContainerId.newContainerId(appAttemptId, 2), ContainerState.COMPLETE, "", 0);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs2, anyNodeId));
    assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    boolean found = false;
    for (ContainerStatus containerStatus : applicationAttempt.getJustFinishedContainers()) {
        if (cs2.getContainerId().equals(containerStatus.getContainerId())) {
            found = true;
        }
    }
    assertTrue(found);
}

Example 9 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class RMAppAttemptImpl method rememberTargetTransitionsAndStoreState.

private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, Object transitionToDo, RMAppAttemptState targetFinalState, RMAppAttemptState stateToBeStored) {
    rememberTargetTransitions(event, transitionToDo, targetFinalState);
    stateBeforeFinalSaving = getState();
    // As of today, finalState, diagnostics, final-tracking-url and
    // finalAppStatus are the only things that we store into the StateStore
    // AFTER the initial saving on app-attempt-start
    // These fields can be visible from outside only after they are saved in
    // StateStore
    String diags = null;
    // don't leave the tracking URL pointing to a non-existent AM
    if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
        setTrackingUrlToAHSPage(stateToBeStored);
    } else {
        setTrackingUrlToRMAppPage(stateToBeStored);
    }
    String finalTrackingUrl = getOriginalTrackingUrl();
    FinalApplicationStatus finalStatus = null;
    int exitStatus = ContainerExitStatus.INVALID;
    switch(event.getType()) {
        case LAUNCH_FAILED:
            diags = event.getDiagnosticMsg();
            break;
        case REGISTERED:
            diags = getUnexpectedAMRegisteredDiagnostics();
            break;
        case UNREGISTERED:
            RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event;
            diags = unregisterEvent.getDiagnosticMsg();
            // reset finalTrackingUrl to url sent by am
            finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
            finalStatus = unregisterEvent.getFinalApplicationStatus();
            break;
        case CONTAINER_FINISHED:
            RMAppAttemptContainerFinishedEvent finishEvent = (RMAppAttemptContainerFinishedEvent) event;
            diags = getAMContainerCrashedDiagnostics(finishEvent);
            exitStatus = finishEvent.getContainerStatus().getExitStatus();
            break;
        case KILL:
            break;
        case FAIL:
            diags = event.getDiagnosticMsg();
            break;
        case EXPIRE:
            diags = getAMExpiredDiagnostics(event);
            break;
        default:
            break;
    }
    AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage();
    RMStateStore rmStore = rmContext.getStateStore();
    setFinishTime(System.currentTimeMillis());
    ApplicationAttemptStateData attemptState = ApplicationAttemptStateData.newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus, getFinishTime(), resUsage.getMemorySeconds(), resUsage.getVcoreSeconds(), this.attemptMetrics.getPreemptedMemory(), this.attemptMetrics.getPreemptedVcore());
    LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus);
    rmStore.updateApplicationAttemptState(attemptState);
}

Example 10 with RMAppAttemptContainerFinishedEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent in project hadoop by apache.

the class TestAMRMTokens method testTokenExpiry.

/**
   * Validate that application tokens are unusable after the
   * application-finishes.
   * 
   * @throws Exception
   */
@SuppressWarnings("unchecked")
@Test
public void testTokenExpiry() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationMasterProtocol rmClient = null;
    try {
        MockNM nm1 = rm.registerNode("localhost:1234", 5120);
        RMApp app = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        int waitCount = 0;
        while (containerManager.containerTokens == null && waitCount++ < 20) {
            LOG.info("Waiting for AM Launch to happen..");
            Thread.sleep(1000);
        }
        Assert.assertNotNull(containerManager.containerTokens);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
        // Create a client to the RM.
        UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
        Credentials credentials = containerManager.getContainerCredentials();
        final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
        Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
        currentUser.addToken(amRMToken);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
        rmClient.registerApplicationMaster(request);
        FinishApplicationMasterRequest finishAMRequest = Records.newRecord(FinishApplicationMasterRequest.class);
        finishAMRequest.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
        finishAMRequest.setDiagnostics("diagnostics");
        finishAMRequest.setTrackingUrl("url");
        rmClient.finishApplicationMaster(finishAMRequest);
        // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
        // from Finishing state to Finished State. Both AMRMToken and
        // ClientToAMToken will be removed.
        ContainerStatus containerStatus = BuilderUtils.newContainerStatus(attempt.getMasterContainer().getId(), ContainerState.COMPLETE, "AM Container Finished", 0, attempt.getMasterContainer().getResource());
        rm.getRMContext().getDispatcher().getEventHandler().handle(new RMAppAttemptContainerFinishedEvent(applicationAttemptId, containerStatus, nm1.getNodeId()));
        // Make sure the RMAppAttempt is at Finished State.
        // Both AMRMToken and ClientToAMToken have been removed.
        int count = 0;
        while (attempt.getState() != RMAppAttemptState.FINISHED && count < maxWaitAttempts) {
            Thread.sleep(100);
            count++;
        }
        Assert.assertTrue(attempt.getState() == RMAppAttemptState.FINISHED);
        // Now simulate trying to allocate. RPC call itself should throw auth
        // exception.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
        try {
            rmClient.allocate(allocateRequest);
            Assert.fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!");
        } catch (Throwable t) {
            LOG.info("Exception found is ", t);
            // The exception will still have the earlier appAttemptId as it picks it
            // up from the token.
            Assert.assertTrue(t.getCause().getMessage().contains(applicationAttemptId.toString() + " not found in AMRMTokenSecretManager."));
        }
    } finally {
        rm.stop();
        if (rmClient != null) {
            // To avoid using cached client
            rpc.stopProxy(rmClient, conf);
        }
    }
}