Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 11 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitStartedEventFail_OnDAGSuccess.

@Test(timeout = 5000)
public void testDAGCommitStartedEventFail_OnDAGSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    historyEventHandler.failDAGCommitStartedEvent = true;
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(DAGTerminationCause.RECOVERY_FAILURE, dag.getTerminationCause());
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit has not started
    Assert.assertEquals(0, v12OutputCommitter.commitCounter);
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit has not started
    Assert.assertEquals(0, v12OutputCommitter.commitCounter);
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 12 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess.

@Test(timeout = 5000)
public void testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    dag.handle(new DAGEvent(dag.getID(), DAGEventType.INTERNAL_ERROR));
    waitUntil(dag, DAGState.ERROR);
    Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR, dag.getTerminationCause());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 13 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestDAGImpl method testGroupDAGCompletionWithCommitSuccess.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGroupDAGCompletionWithCommitSuccess() {
    // should have only 2 commits. 1 vertex3 commit and 1 group commit.
    initDAG(groupDag);
    startDAG(groupDag);
    dispatcher.await();
    for (int i = 0; i < 3; ++i) {
        Vertex v = groupDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, groupDag.getSuccessfulVertices());
    }
    Assert.assertEquals(3, groupDag.getSuccessfulVertices());
    Assert.assertTrue(1.0f == groupDag.getCompletedTaskProgress());
    Assert.assertEquals(DAGState.SUCCEEDED, groupDag.getState());
    Assert.assertEquals(2, TotalCountingOutputCommitter.totalCommitCounter);
}

Example 14 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestDAGImpl method testVertexReRunning.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexReRunning() {
    initDAG(dag);
    dag.dagScheduler = mock(DAGScheduler.class);
    startDAG(dag);
    dispatcher.await();
    TezVertexID vId = TezVertexID.getInstance(dagId, 1);
    Vertex v = dag.getVertex(vId);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 1), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dag.getSuccessfulVertices());
    Assert.assertEquals(1, dag.numCompletedVertices);
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(TezTaskID.getInstance(vId, 0)));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(0, dag.getSuccessfulVertices());
    Assert.assertEquals(0, dag.numCompletedVertices);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dag.getSuccessfulVertices());
    Assert.assertEquals(1, dag.numCompletedVertices);
}

Example 15 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testVertexCommitFail1_OnVertexSuccess.

// the first commit fail which cause the second commit abort
@Test(timeout = 5000)
public void testVertexCommitFail1_OnVertexSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(false, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    v1OutputCommitter_1.unblockCommit();
    waitUntil(v1, VertexState.FAILED);
    Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v1.getTerminationCause());
    Assert.assertTrue(v1.commitFutures.isEmpty());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.commitCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
    // can't verify the commitCounter because v1OutputCommitter_2 may not be started
    Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}