use of org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter in project tez by apache.
the class TestDAGImpl method testDAGCompletionWithCommitSuccess.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitSuccess() {
// all vertices completed -> DAG completion and commit
initDAG(mrrDag);
dispatcher.await();
startDAG(mrrDag);
dispatcher.await();
for (int i = 0; i < 2; ++i) {
Vertex v = mrrDag.getVertex("vertex" + (i + 1));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
}
// no commit yet
for (Vertex v : mrrDag.vertices.values()) {
for (OutputCommitter c : v.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
// dag completion and commit
Vertex v = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
Assert.assertEquals(DAGState.SUCCEEDED, mrrDag.getState());
for (Vertex vertex : mrrDag.vertices.values()) {
for (OutputCommitter c : vertex.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
}
use of org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter in project tez by apache.
the class TestDAGImpl method testDAGErrorAbortNonSuccessfulOutputs.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortNonSuccessfulOutputs() {
// vertex success -> vertex output commit. failed dag aborts only non-successful vertices
mrrDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
initDAG(mrrDag);
dispatcher.await();
startDAG(mrrDag);
dispatcher.await();
for (int i = 0; i < 2; ++i) {
Vertex v = mrrDag.getVertex("vertex" + (i + 1));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
for (OutputCommitter c : v.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
// error on vertex -> dag error
Vertex errorVertex = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEvent(errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
dispatcher.await();
Assert.assertEquals(VertexState.ERROR, errorVertex.getState());
dispatcher.await();
Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
for (Vertex vertex : mrrDag.vertices.values()) {
for (OutputCommitter c : vertex.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
if (vertex == errorVertex) {
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
} else {
// abort operation should take no side effort on the successful commit
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
}
}
use of org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter in project tez by apache.
the class TestDAGImpl method testDAGErrorAbortAllOutputs.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortAllOutputs() {
// error on a vertex -> dag error -> all outputs aborted.
initDAG(mrrDag);
dispatcher.await();
startDAG(mrrDag);
dispatcher.await();
for (int i = 0; i < 2; ++i) {
Vertex v = mrrDag.getVertex("vertex" + (i + 1));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
}
// no commit yet
for (Vertex v : mrrDag.vertices.values()) {
for (OutputCommitter c : v.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
// vertex error -> dag error -> abort all outputs
Vertex v = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
dispatcher.await();
Assert.assertEquals(VertexState.ERROR, v.getState());
Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
for (Vertex vertex : mrrDag.vertices.values()) {
for (OutputCommitter c : vertex.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
}
use of org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter in project tez by apache.
the class TestDAGImpl method testDAGCompletionWithCommitFailure.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitFailure() throws IOException {
// all vertices completed -> DAG completion and commit
initDAG(mrrDag);
dispatcher.await();
// committer for bad vertex will throw exception
Vertex badVertex = mrrDag.getVertex("vertex3");
List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(true, false, false).toUserPayload())).build())).setName("output3").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
badVertex.setAdditionalOutputs(outputs);
startDAG(mrrDag);
dispatcher.await();
for (int i = 0; i < 2; ++i) {
Vertex v = mrrDag.getVertex("vertex" + (i + 1));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
}
// no commit yet
for (Vertex v : mrrDag.vertices.values()) {
for (OutputCommitter c : v.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
// dag completion and commit. Exception causes all outputs to be aborted
Vertex v = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
Assert.assertEquals(DAGState.FAILED, mrrDag.getState());
Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, mrrDag.getTerminationCause());
for (Vertex vertex : mrrDag.vertices.values()) {
for (OutputCommitter c : vertex.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
}
Aggregations