use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.
the class TestCommit method testDAGCommitFail4_OnVertexSuccess.
// commit of vertex v3 succeeded first and then commit of vertex group(v1,v2) fail
@Test(timeout = 5000)
public void testDAGCommitFail4_OnVertexSuccess() throws Exception {
conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
setupDAG(createDAGPlan(false, true));
initDAG(dag);
startDAG(dag);
VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
Assert.assertEquals(VertexState.COMMITTING, v3.getState());
Assert.assertEquals(DAGState.RUNNING, dag.getState());
CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
v3OutputCommitter.unblockCommit();
waitForCommitCompleted(dag, new OutputKey("v3Out", "vertex3", true));
waitUntil(v3, VertexState.SUCCEEDED);
Assert.assertTrue(v3.commitFutures.isEmpty());
CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
v12OutputCommitter.unblockCommit();
waitUntil(dag, DAGState.FAILED);
Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, dag.getTerminationCause());
Assert.assertTrue(dag.commitFutures.isEmpty());
historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
Assert.assertEquals(1, v12OutputCommitter.initCounter);
Assert.assertEquals(1, v12OutputCommitter.setupCounter);
Assert.assertEquals(1, v12OutputCommitter.commitCounter);
Assert.assertEquals(1, v12OutputCommitter.abortCounter);
Assert.assertEquals(1, v3OutputCommitter.initCounter);
Assert.assertEquals(1, v3OutputCommitter.setupCounter);
Assert.assertEquals(1, v3OutputCommitter.commitCounter);
Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}
use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.
the class TestCommit method testDAGCommitFail1_OnVertexSuccess.
// commit of vertex group(v1,v2) fail and commit of v3 is not completed
@Test(timeout = 5000)
public void testDAGCommitFail1_OnVertexSuccess() throws Exception {
conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
setupDAG(createDAGPlan(false, true));
initDAG(dag);
startDAG(dag);
VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
Assert.assertEquals(VertexState.COMMITTING, v3.getState());
Assert.assertEquals(DAGState.RUNNING, dag.getState());
CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
v12OutputCommitter.unblockCommit();
waitUntil(dag, DAGState.FAILED);
// v3 is killed due to the commit failure of the vertex group (v1,v2)
Assert.assertEquals(VertexState.KILLED, v3.getState());
Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE, v3.getTerminationCause());
Assert.assertTrue(v3.commitFutures.isEmpty());
Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, dag.getTerminationCause());
Assert.assertTrue(dag.commitFutures.isEmpty());
historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
Assert.assertEquals(1, v12OutputCommitter.initCounter);
Assert.assertEquals(1, v12OutputCommitter.setupCounter);
Assert.assertEquals(1, v12OutputCommitter.commitCounter);
Assert.assertEquals(1, v12OutputCommitter.abortCounter);
CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
Assert.assertEquals(1, v3OutputCommitter.initCounter);
Assert.assertEquals(1, v3OutputCommitter.setupCounter);
// commit may not have started, so can't verify commitCounter
Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}
use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.
the class TestCommit method testDAGCommitSucceeded1_OnVertexSuccess.
// commit of v3Out complete first then commit of v12Out complete
@Test(timeout = 5000)
public void testDAGCommitSucceeded1_OnVertexSuccess() throws Exception {
conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
setupDAG(createDAGPlan(true, true));
initDAG(dag);
startDAG(dag);
VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
Assert.assertEquals(VertexState.COMMITTING, v3.getState());
Assert.assertEquals(DAGState.RUNNING, dag.getState());
CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
v3OutputCommitter.unblockCommit();
waitUntil(v3, VertexState.SUCCEEDED);
// dag go to COMMITTING due to the pending vertex group commit of v1,v2
waitUntil(dag, DAGState.COMMITTING);
CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
v12OutputCommitter.unblockCommit();
waitUntil(dag, DAGState.SUCCEEDED);
Assert.assertTrue(dag.commitFutures.isEmpty());
historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitStartedEvent("v1", 0);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("v1", 0);
historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
Assert.assertEquals(1, v12OutputCommitter.initCounter);
Assert.assertEquals(1, v12OutputCommitter.setupCounter);
Assert.assertEquals(1, v12OutputCommitter.commitCounter);
Assert.assertEquals(0, v12OutputCommitter.abortCounter);
Assert.assertEquals(1, v3OutputCommitter.initCounter);
Assert.assertEquals(1, v3OutputCommitter.setupCounter);
Assert.assertEquals(1, v3OutputCommitter.commitCounter);
Assert.assertEquals(0, v3OutputCommitter.abortCounter);
}
use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.
the class TestCommit method testDAGCommitFail3_OnVertexSuccess.
// vertex group (v1,v2) succeeded first and then commit of vertex v3 fail
@Test(timeout = 5000)
public void testDAGCommitFail3_OnVertexSuccess() throws Exception {
conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
setupDAG(createDAGPlan(true, false));
initDAG(dag);
startDAG(dag);
VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
Assert.assertEquals(VertexState.COMMITTING, v3.getState());
Assert.assertEquals(DAGState.RUNNING, dag.getState());
CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
v12OutputCommitter.unblockCommit();
waitForCommitCompleted(dag, new OutputKey("v12Out", "uv12", true));
CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
v3OutputCommitter.unblockCommit();
waitUntil(dag, DAGState.FAILED);
Assert.assertEquals(VertexState.FAILED, v3.getState());
Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v3.getTerminationCause());
Assert.assertTrue(v3.commitFutures.isEmpty());
Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
Assert.assertTrue(dag.commitFutures.isEmpty());
historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 1);
historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
Assert.assertEquals(1, v12OutputCommitter.initCounter);
Assert.assertEquals(1, v12OutputCommitter.setupCounter);
Assert.assertEquals(1, v12OutputCommitter.commitCounter);
Assert.assertEquals(1, v12OutputCommitter.abortCounter);
Assert.assertEquals(1, v3OutputCommitter.initCounter);
Assert.assertEquals(1, v3OutputCommitter.setupCounter);
Assert.assertEquals(1, v3OutputCommitter.commitCounter);
Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}
use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.
the class TestCommit method testVertexGroupCommitFinishedEventFail_OnVertexSuccess.
@Test(timeout = 5000)
public void testVertexGroupCommitFinishedEventFail_OnVertexSuccess() throws Exception {
conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
setupDAG(createDAGPlan(true, true));
historyEventHandler.failVertexGroupCommitFinishedEvent = true;
initDAG(dag);
startDAG(dag);
VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
// need to make vertices to go to SUCCEEDED
v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
v12OutputCommitter.unblockCommit();
waitUntil(dag, DAGState.FAILED);
historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 1);
historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 0);
historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
Assert.assertEquals(DAGState.FAILED, dag.getState());
Assert.assertEquals(DAGTerminationCause.RECOVERY_FAILURE, dag.getTerminationCause());
Assert.assertTrue(dag.commitFutures.isEmpty());
Assert.assertEquals(VertexState.KILLED, v3.getState());
Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE, v3.getTerminationCause());
Assert.assertTrue(v3.commitFutures.isEmpty());
Assert.assertEquals(1, v12OutputCommitter.initCounter);
Assert.assertEquals(1, v12OutputCommitter.setupCounter);
Assert.assertEquals(1, v12OutputCommitter.commitCounter);
Assert.assertEquals(1, v12OutputCommitter.abortCounter);
Assert.assertEquals(1, v3OutputCommitter.initCounter);
Assert.assertEquals(1, v3OutputCommitter.setupCounter);
// commit may not have started, so can't verify commitCounter
Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}
Aggregations