Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 56 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testDAGErrorAbortAllOutputs.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortAllOutputs() {
    // error on a vertex -> dag error -> all outputs aborted.
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // vertex error -> dag error -> abort all outputs
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.ERROR, v.getState());
    Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(1, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}

Example 57 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method _testTerminateRunningDAG.

@SuppressWarnings("unchecked")
private void _testTerminateRunningDAG(DAGTerminationCause terminationCause) {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();
    TezVertexID vId1 = TezVertexID.getInstance(dagId, 1);
    Vertex v1 = dag.getVertex(vId1);
    ((EventHandler<VertexEvent>) v1).handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId1, 0), TaskState.SUCCEEDED));
    TezVertexID vId0 = TezVertexID.getInstance(dagId, 0);
    Vertex v0 = dag.getVertex(vId0);
    ((EventHandler<VertexEvent>) v0).handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId0, 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    dispatcher.getEventHandler().handle(new DAGEventTerminateDag(dagId, terminationCause, null));
    dispatcher.await();
    Assert.assertEquals(DAGState.TERMINATING, dag.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
    Assert.assertEquals(VertexState.TERMINATING, v1.getState());
    for (int i = 2; i < 6; ++i) {
        TezVertexID vId = TezVertexID.getInstance(dagId, i);
        Vertex v = dag.getVertex(vId);
        Assert.assertEquals(VertexState.KILLED, v.getState());
    }
    Assert.assertEquals(1, dag.getSuccessfulVertices());
}

Example 58 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGImpl method testDAGCompletionWithCommitFailure.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitFailure() throws IOException {
    // all vertices completed -> DAG completion and commit
    initDAG(mrrDag);
    dispatcher.await();
    // committer for bad vertex will throw exception
    Vertex badVertex = mrrDag.getVertex("vertex3");
    List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
    outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(true, false, false).toUserPayload())).build())).setName("output3").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
    badVertex.setAdditionalOutputs(outputs);
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // dag completion and commit. Exception causes all outputs to be aborted
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.FAILED, mrrDag.getState());
    Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, mrrDag.getTerminationCause());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(1, committer.abortCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}

Example 59 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestAMWebController method testGetVerticesInfoGetAll.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGetVerticesInfoGetAll() {
    Vertex mockVertex1 = createMockVertex("vertex_1422960590892_0007_42_00", VertexState.RUNNING, 0.33f, 3);
    Vertex mockVertex2 = createMockVertex("vertex_1422960590892_0007_42_01", VertexState.SUCCEEDED, 1.0f, 5);
    final Map<String, Object> result = getVerticesTestHelper(0, mockVertex1, mockVertex2);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.containsKey("vertices"));
    ArrayList<Map<String, String>> verticesInfo = (ArrayList<Map<String, String>>) result.get("vertices");
    Assert.assertEquals(2, verticesInfo.size());
    Map<String, String> vertex1Result = verticesInfo.get(0);
    Map<String, String> vertex2Result = verticesInfo.get(1);
    verifySingleVertexResult(mockVertex1, vertex1Result);
    verifySingleVertexResult(mockVertex2, vertex2Result);
}

Example 60 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestAMWebController method createMockVertex.

private Vertex createMockVertex(String vertexIDStr, VertexState status, float progress, int taskCounts) {
    ProgressBuilder pb = new ProgressBuilder();
    pb.setTotalTaskCount(taskCounts);
    pb.setSucceededTaskCount(taskCounts * 2);
    pb.setFailedTaskAttemptCount(taskCounts * 3);
    pb.setKilledTaskAttemptCount(taskCounts * 4);
    pb.setRunningTaskCount(taskCounts * 5);
    Vertex mockVertex = mock(Vertex.class);
    doReturn(TezVertexID.fromString(vertexIDStr)).when(mockVertex).getVertexId();
    doReturn(status).when(mockVertex).getState();
    doReturn(progress).when(mockVertex).getProgress();
    doReturn(pb).when(mockVertex).getVertexProgress();
    doReturn(1L).when(mockVertex).getInitTime();
    doReturn(1L).when(mockVertex).getStartTime();
    doReturn(2L).when(mockVertex).getFinishTime();
    doReturn(1L).when(mockVertex).getFirstTaskStartTime();
    doReturn(2L).when(mockVertex).getLastTaskFinishTime();
    TezCounters counters = new TezCounters();
    counters.addGroup("g1", "g1");
    counters.addGroup("g2", "g2");
    counters.addGroup("g3", "g3");
    counters.addGroup("g4", "g4");
    counters.findCounter("g1", "g1_c1").setValue(100);
    counters.findCounter("g1", "g1_c2").setValue(100);
    counters.findCounter("g2", "g2_c3").setValue(100);
    counters.findCounter("g2", "g2_c4").setValue(100);
    counters.findCounter("g3", "g3_c5").setValue(100);
    counters.findCounter("g3", "g3_c6").setValue(100);
    doReturn(counters).when(mockVertex).getAllCounters();
    doReturn(counters).when(mockVertex).getCachedCounters();
    return mockVertex;
}