Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 26 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestTaskImpl method testFailedTaskTransitionWithLaunchedAttempt.

@Test(timeout = 30000)
public void testFailedTaskTransitionWithLaunchedAttempt() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.NO_PROGRESS));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.NO_PROGRESS));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    secondMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, secondMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    mockTask.handle(new TaskEventTAFailed(secondMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    assertTrue("Attempts should have failed!", firstMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED);
    assertEquals("Task should have no uncompleted attempts!", 0, mockTask.getUncompletedAttemptsCount());
    assertTrue("Task should have failed!", mockTask.getState() == TaskState.FAILED);
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl thirdMockTaskAttempt = mockTask.getLastAttempt();
    mockTask.handle(createTaskTALauncherEvent(thirdMockTaskAttempt.getID()));
}

Example 27 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testVertexFailureHandling.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexFailureHandling() {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 0), VertexState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 1), VertexState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 2), VertexState.FAILED));
    dispatcher.await();
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(2, dag.getSuccessfulVertices());
    // Expect running vertices to be killed on first failure
    for (int i = 3; i < 6; ++i) {
        TezVertexID vId = TezVertexID.getInstance(dagId, i);
        Vertex v = dag.getVertex(vId);
        Assert.assertEquals(VertexState.KILLED, v.getState());
    }
}

Example 28 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testGroupDAGWithVertexReRunning.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGroupDAGWithVertexReRunning() {
    groupDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    initDAG(groupDag);
    startDAG(groupDag);
    dispatcher.await();
    Vertex v1 = groupDag.getVertex("vertex1");
    Vertex v2 = groupDag.getVertex("vertex2");
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new DAGEventVertexReRunning(v1.getVertexId()));
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v2.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    // commit should not happen due to vertex-rerunning
    Assert.assertEquals(0, TotalCountingOutputCommitter.totalCommitCounter);
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    // commit happen
    Assert.assertEquals(1, TotalCountingOutputCommitter.totalCommitCounter);
    Assert.assertEquals(2, groupDag.getSuccessfulVertices());
}

Example 29 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testKillStartedDAG.

@SuppressWarnings("unchecked")
public void testKillStartedDAG() {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();
    dispatcher.getEventHandler().handle(new DAGEventTerminateDag(dagId, DAGTerminationCause.DAG_KILL, null));
    dispatcher.await();
    Assert.assertEquals(DAGState.KILLED, dag.getState());
    for (int i = 0; i < 6; ++i) {
        TezVertexID vId = TezVertexID.getInstance(dagId, i);
        Vertex v = dag.getVertex(vId);
        Assert.assertEquals(VertexState.KILLED, v.getState());
    }
}

Example 30 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testDAGCompletionWithCommitSuccess.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitSuccess() {
    // all vertices completed -> DAG completion and commit
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // dag completion and commit
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.SUCCEEDED, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(1, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}