Examples with NodeId org.apache.hadoop.yarn.api.records.NodeId used on opensource projects

Example 51 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testUsageReport.

@Test
public void testUsageReport() {
    // scheduler has info on running apps
    ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
    ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class);
    when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
    when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
    // start and finish the attempt
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId, "", FinalApplicationStatus.SUCCEEDED, ""));
    // expect usage stats to come from the scheduler report
    ApplicationResourceUsageReport report = applicationAttempt.getApplicationResourceUsageReport();
    Assert.assertEquals(123456L, report.getMemorySeconds());
    Assert.assertEquals(55544L, report.getVcoreSeconds());
    // finish app attempt and remove it from scheduler 
    when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
    sendAttemptUpdateSavedEvent(applicationAttempt);
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(attemptId, ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
    when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
    report = applicationAttempt.getApplicationResourceUsageReport();
    Assert.assertEquals(223456, report.getMemorySeconds());
    Assert.assertEquals(75544, report.getVcoreSeconds());
}

Example 52 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testRunningToFailedUserTriggeredFailEvent.

@Test(timeout = 30000)
public void testRunningToFailedUserTriggeredFailEvent() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.FAIL, FAILED_DIAGNOSTICS));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    assertEquals(amContainer, applicationAttempt.getMasterContainer());
    assertEquals(0, application.getRanNodes().size());
    String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app", applicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
    verifyAMHostAndPortInvalidated();
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}

Example 53 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinalSavingToFinishedWithContainerFinished.

// While attempt is at FINAL_SAVING, Contaienr_Finished event may come before
// Attempt_Saved event, we stay on FINAL_SAVING on Container_Finished event
// and then directly jump from FINAL_SAVING to FINISHED state on Attempt_Saved
// event
@Test
public void testFinalSavingToFinishedWithContainerFinished() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
    String trackingUrl = "mytrackingurl";
    String diagnostics = "Successful";
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, diagnostics));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    // Container_finished event comes before Attempt_Saved event.
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    // send attempt_saved
    sendAttemptUpdateSavedEvent(applicationAttempt);
    testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, diagnostics, 0, false);
}

Example 54 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testLaunchedFailWhileAHSEnabled.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testLaunchedFailWhileAHSEnabled() {
    Configuration myConf = new Configuration(conf);
    myConf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
    ApplicationId applicationId = MockApps.newAppID(appId);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 2);
    RMAppAttempt myApplicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, myConf, BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    //submit, schedule and allocate app attempt
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.START));
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.ATTEMPT_ADDED));
    Container amContainer = mock(Container.class);
    Resource resource = BuilderUtils.newResource(2048, 1);
    when(amContainer.getId()).thenReturn(BuilderUtils.newContainerId(myApplicationAttempt.getAppAttemptId(), 1));
    when(amContainer.getResource()).thenReturn(resource);
    Allocation allocation = mock(Allocation.class);
    when(allocation.getContainers()).thenReturn(Collections.singletonList(amContainer));
    when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class))).thenReturn(allocation);
    RMContainer rmContainer = mock(RMContainerImpl.class);
    when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer);
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.CONTAINER_ALLOCATED));
    assertEquals(RMAppAttemptState.ALLOCATED_SAVING, myApplicationAttempt.getAppAttemptState());
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
    // launch app attempt
    myApplicationAttempt.handle(new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), RMAppAttemptEventType.LAUNCHED));
    assertEquals(YarnApplicationAttemptState.LAUNCHED, myApplicationAttempt.createApplicationAttemptState());
    //fail container right after launched
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    myApplicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(myApplicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    sendAttemptUpdateSavedEvent(myApplicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, myApplicationAttempt.getAppAttemptState());
    String rmAppPageUrl = pjoin(AHS_WEBAPP_ADDR, "applicationhistory", "app", myApplicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, myApplicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, myApplicationAttempt.getTrackingUrl());
}

Example 55 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFailedToFailed.

@Test
public void testFailedToFailed() {
    // create a failed attempt.
    when(submissionContext.getKeepContainersAcrossApplicationAttempts()).thenReturn(true);
    when(application.getMaxAppAttempts()).thenReturn(2);
    when(application.getNumFailedAppAttempts()).thenReturn(1);
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123);
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs1, anyNodeId));
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    // should not kill containers when attempt fails.
    assertTrue(transferStateFromPreviousAttempt);
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
    // failed attempt captured the container finished event.
    assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
    ContainerStatus cs2 = ContainerStatus.newInstance(ContainerId.newContainerId(appAttemptId, 2), ContainerState.COMPLETE, "", 0);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs2, anyNodeId));
    assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    boolean found = false;
    for (ContainerStatus containerStatus : applicationAttempt.getJustFinishedContainers()) {
        if (cs2.getContainerId().equals(containerStatus.getContainerId())) {
            found = true;
        }
    }
    assertTrue(found);
}