Examples with TaskAttemptImpl org.apache.tez.dag.app.dag.impl.TaskAttemptImpl used on opensource projects

Example 1 with TaskAttemptImpl

use of org.apache.tez.dag.app.dag.impl.TaskAttemptImpl in project tez by apache.

the class TestTaskSchedulerManager method testContainerDiskFailed.

@Test(timeout = 5000)
public void testContainerDiskFailed() throws IOException {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);
    schedulerHandler.start();
    String diagnostics = "NM disk failed.";
    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
    ContainerStatus mockStatus = mock(ContainerStatus.class);
    ContainerId mockCId = mock(ContainerId.class);
    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
    assertEquals(1, mockEventHandler.events.size());
    Event event = mockEventHandler.events.get(0);
    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
    assertEquals(mockCId, completedEvent.getContainerId());
    assertEquals("Container disk failed. NM disk failed.", completedEvent.getDiagnostics());
    Assert.assertFalse(completedEvent.isPreempted());
    assertTrue(completedEvent.isDiskFailed());
    assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR, completedEvent.getTerminationCause());
    schedulerHandler.stop();
    schedulerHandler.close();
}

Example 2 with TaskAttemptImpl

use of org.apache.tez.dag.app.dag.impl.TaskAttemptImpl in project tez by apache.

the class TestPreemption method testPreemptionJob.

void testPreemptionJob(MockTezClient tezClient, DAG dag, int vertexIndex, int upToTaskVersion, String info) throws Exception {
    System.out.println("TestPreemption - Running - " + info);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    // turn off scheduling to block DAG before submitting it
    mockLauncher.startScheduling(false);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
        Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, taImpl.getTerminationCause());
    }
    System.out.println("TestPreemption - Done running - " + info);
}

Example 3 with TaskAttemptImpl

use of org.apache.tez.dag.app.dag.impl.TaskAttemptImpl in project tez by apache.

the class TestPreemption method testPreemptionWithoutSession.

@Test(timeout = 5000)
public void testPreemptionWithoutSession() throws Exception {
    System.out.println("TestPreemptionWithoutSession");
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    AtomicBoolean mockAppLauncherGoFlag = new AtomicBoolean(false);
    MockTezClient tezClient = new MockTezClient("testPreemption", tezconf, false, null, null, null, mockAppLauncherGoFlag, false, false, 2, 2);
    tezClient.start();
    DAGClient dagClient = tezClient.submitDAG(createDAG(DataMovementType.SCATTER_GATHER));
    // now the MockApp has been started. sync with it to get the launcher
    syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient);
    DAGImpl dagImpl;
    do {
        // usually needs to sleep 2-3 times
        Thread.sleep(100);
    } while ((dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG()) == null);
    int vertexIndex = 0;
    int upToTaskVersion = 3;
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
    }
    tezClient.stop();
}

Example 4 with TaskAttemptImpl

use of org.apache.tez.dag.app.dag.impl.TaskAttemptImpl in project tez by apache.

the class TestTaskSchedulerManager method testTaskBasedAffinity.

@Test(timeout = 5000)
public void testTaskBasedAffinity() throws Exception {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);
    schedulerHandler.start();
    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
    TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
    String affVertexName = "srcVertex";
    int affTaskIndex = 1;
    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
    VertexImpl affVertex = mock(VertexImpl.class);
    TaskImpl affTask = mock(TaskImpl.class);
    TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
    ContainerId affCId = mock(ContainerId.class);
    when(affVertex.getTotalTasks()).thenReturn(2);
    when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
    when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
    when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
    when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
    Resource resource = Resource.newInstance(100, 1);
    AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest(taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
    schedulerHandler.notify.set(false);
    schedulerHandler.handle(event);
    synchronized (schedulerHandler.notify) {
        while (!schedulerHandler.notify.get()) {
            schedulerHandler.notify.wait();
        }
    }
    // verify mockTaskAttempt affinitized to expected affCId
    verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId, Priority.newInstance(3), null, event);
    schedulerHandler.stop();
    schedulerHandler.close();
}

Example 5 with TaskAttemptImpl

use of org.apache.tez.dag.app.dag.impl.TaskAttemptImpl in project tez by apache.

the class TestTaskSchedulerManager method testSimpleAllocate.

@Test(timeout = 5000)
public void testSimpleAllocate() throws Exception {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);
    schedulerHandler.start();
    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
    TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
    when(mockAttemptId.getId()).thenReturn(0);
    when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
    Resource resource = Resource.newInstance(1024, 1);
    ContainerContext containerContext = new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(), new HashMap<String, String>(), "");
    int priority = 10;
    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
    ContainerId mockCId = mock(ContainerId.class);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(mockCId);
    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint, priority, containerContext, 0, 0, 0);
    schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
    assertEquals(1, mockEventHandler.events.size());
    assertTrue(mockEventHandler.events.get(0) instanceof AMContainerEventAssignTA);
    AMContainerEventAssignTA assignEvent = (AMContainerEventAssignTA) mockEventHandler.events.get(0);
    assertEquals(priority, assignEvent.getPriority());
    assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
}