Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 41 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitFail2_OnVertexSuccess.

// commit of vertex v3 fail and commit of vertex group (v1,v2) is not completed
@Test(timeout = 5000)
public void testDAGCommitFail2_OnVertexSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan(true, false));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    Assert.assertEquals(VertexState.COMMITTING, v3.getState());
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    v3OutputCommitter.unblockCommit();
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(VertexState.FAILED, v3.getState());
    Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v3.getTerminationCause());
    Assert.assertTrue(v3.commitFutures.isEmpty());
    Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    Assert.assertEquals(1, v3OutputCommitter.commitCounter);
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 42 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method _testDAGTerminatedWhileCommitting1_OnDAGSuccess.

// Kill dag while it is in COMMITTING in the case of
// TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is true
private void _testDAGTerminatedWhileCommitting1_OnDAGSuccess(DAGTerminationCause terminationCause) throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    dag.handle(new DAGEventTerminateDag(dag.getID(), terminationCause, null));
    waitUntil(dag, terminationCause.getFinishedState());
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
    Assert.assertEquals(terminationCause, dag.getTerminationCause());
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 43 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testVertexRescheduleWhileCommitting.

@Test(timeout = 5000)
public void testVertexRescheduleWhileCommitting() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    // reschedule task
    v1.handle(new VertexEventTaskReschedule(TezTaskID.getInstance(v1.getVertexId(), 2)));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING, v1.getTerminationCause());
    Assert.assertTrue(v1.commitFutures.isEmpty());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}

Example 44 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitSucceeded3_OnVertexSuccess.

// test DAGCommitSucceeded when vertex group has multiple shared outputs
@Test(timeout = 5000)
public void testDAGCommitSucceeded3_OnVertexSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlanWith2VertexGroupOutputs(true, true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    Assert.assertEquals(VertexState.COMMITTING, v3.getState());
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    CountingOutputCommitter v12OutputCommitter1 = (CountingOutputCommitter) v1.getOutputCommitter("v12Out1");
    v12OutputCommitter1.unblockCommit();
    CountingOutputCommitter v12OutputCommitter2 = (CountingOutputCommitter) v1.getOutputCommitter("v12Out2");
    v12OutputCommitter2.unblockCommit();
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    v3OutputCommitter.unblockCommit();
    waitUntil(dag, DAGState.SUCCEEDED);
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 1);
    historyEventHandler.verifyVertexGroupCommitStartedEvent("v1", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("v1", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    Assert.assertEquals(1, v12OutputCommitter1.initCounter);
    Assert.assertEquals(1, v12OutputCommitter1.setupCounter);
    Assert.assertEquals(1, v12OutputCommitter1.commitCounter);
    Assert.assertEquals(0, v12OutputCommitter1.abortCounter);
    Assert.assertEquals(1, v12OutputCommitter2.initCounter);
    Assert.assertEquals(1, v12OutputCommitter2.setupCounter);
    Assert.assertEquals(1, v12OutputCommitter2.commitCounter);
    Assert.assertEquals(0, v12OutputCommitter2.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    Assert.assertEquals(1, v3OutputCommitter.commitCounter);
    Assert.assertEquals(0, v3OutputCommitter.abortCounter);
}

Example 45 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method _testDAGTerminatedWhileCommitting1_OnVertexSuccess.

// Kill dag while it is in COMMITTING in the case of
// TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is false
private void _testDAGTerminatedWhileCommitting1_OnVertexSuccess(DAGTerminationCause terminationCause) throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v3.getState());
    // dag is still in RUNNING because v3 has not completed
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    v3OutputCommitter.unblockCommit();
    // dag go to COMMITTING due to the pending commit of v12Out
    waitUntil(dag, DAGState.COMMITTING);
    dag.handle(new DAGEventTerminateDag(dag.getID(), terminationCause, null));
    waitUntil(dag, terminationCause.getFinishedState());
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v3.getState());
    Assert.assertEquals(terminationCause.getFinishedState(), dag.getState());
    Assert.assertEquals(terminationCause, dag.getTerminationCause());
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    Assert.assertEquals(1, v3OutputCommitter.commitCounter);
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}