Examples with DAGEventVertexCompleted org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted used on opensource projects

Example 6 with DAGEventVertexCompleted

use of org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted in project tez by apache.

the class TestDAGImpl method testGroupDAGWithVertexReRunning.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGroupDAGWithVertexReRunning() {
    groupDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    initDAG(groupDag);
    startDAG(groupDag);
    dispatcher.await();
    Vertex v1 = groupDag.getVertex("vertex1");
    Vertex v2 = groupDag.getVertex("vertex2");
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new DAGEventVertexReRunning(v1.getVertexId()));
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v2.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    // commit should not happen due to vertex-rerunning
    Assert.assertEquals(0, TotalCountingOutputCommitter.totalCommitCounter);
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    // commit happen
    Assert.assertEquals(1, TotalCountingOutputCommitter.totalCommitCounter);
    Assert.assertEquals(2, groupDag.getSuccessfulVertices());
}

Example 7 with DAGEventVertexCompleted

use of org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted in project tez by apache.

the class TestDAGImpl method _testDAGTerminate.

// Couple of vertices succeed. DAG_KILLED processed, which causes the rest of the vertices to be
// marked as KILLED.
@SuppressWarnings("unchecked")
private void _testDAGTerminate(DAGTerminationCause terminationCause) {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 0), VertexState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 1), VertexState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new DAGEventTerminateDag(dagId, terminationCause, null));
    dispatcher.await();
    Assert.assertEquals(terminationCause.getFinishedState(), dag.getState());
    Assert.assertEquals(terminationCause, dag.getTerminationCause());
    Assert.assertEquals(2, dag.getSuccessfulVertices());
    int killedCount = 0;
    for (Map.Entry<TezVertexID, Vertex> vEntry : dag.getVertices().entrySet()) {
        if (vEntry.getValue().getState() == VertexState.KILLED) {
            killedCount++;
        }
    }
    Assert.assertEquals(4, killedCount);
    for (Vertex v : dag.getVertices().values()) {
        Assert.assertEquals(VertexTerminationCause.DAG_TERMINATED, v.getTerminationCause());
    }
    Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
}

Example 8 with DAGEventVertexCompleted

use of org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted in project tez by apache.

the class TestDAGImpl method testGroupDAGWithVertexReRunningAfterCommit.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGroupDAGWithVertexReRunningAfterCommit() {
    groupDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    initDAG(groupDag);
    startDAG(groupDag);
    dispatcher.await();
    Vertex v1 = groupDag.getVertex("vertex1");
    Vertex v2 = groupDag.getVertex("vertex2");
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(v2.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    // vertex group commit happens
    Assert.assertEquals(1, TotalCountingOutputCommitter.totalCommitCounter);
    // dag failed when vertex re-run happens after vertex group commit is done.
    dispatcher.getEventHandler().handle(new DAGEventVertexReRunning(v1.getVertexId()));
    dispatcher.await();
    Assert.assertEquals(DAGState.FAILED, groupDag.getState());
    Assert.assertEquals(DAGTerminationCause.VERTEX_RERUN_AFTER_COMMIT, groupDag.getTerminationCause());
}

Example 9 with DAGEventVertexCompleted

use of org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted in project tez by apache.

the class TestDAGImpl method testGetDAGStatusWithWait.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testGetDAGStatusWithWait() throws TezException {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();
    // All vertices except one succeed
    for (int i = 0; i < dag.getVertices().size() - 1; ++i) {
        dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, i), VertexState.SUCCEEDED));
    }
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dag.getState());
    Assert.assertEquals(5, dag.getSuccessfulVertices());
    long dagStatusStartTime = System.currentTimeMillis();
    DAGStatusBuilder dagStatus = dag.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000l);
    long dagStatusEndTime = System.currentTimeMillis();
    long diff = dagStatusEndTime - dagStatusStartTime;
    Assert.assertTrue(diff > 1500 && diff < 2500);
    Assert.assertEquals(DAGStatusBuilder.State.RUNNING, dagStatus.getState());
}

Example 10 with DAGEventVertexCompleted

use of org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted in project tez by apache.

the class VertexImpl method finished.

VertexState finished(VertexState finalState, VertexTerminationCause termCause, String diag) {
    if (finishTime == 0)
        setFinishTime();
    if (termCause != null) {
        trySetTerminationCause(termCause);
    }
    if (rootInputInitializerManager != null) {
        rootInputInitializerManager.shutdown();
        rootInputInitializerManager = null;
    }
    switch(finalState) {
        case ERROR:
            addDiagnostic("Vertex: " + logIdentifier + " error due to:" + terminationCause);
            if (!StringUtils.isEmpty(diag)) {
                addDiagnostic(diag);
            }
            abortVertex(VertexStatus.State.valueOf(finalState.name()));
            eventHandler.handle(new DAGEvent(getDAGId(), DAGEventType.INTERNAL_ERROR));
            try {
                logJobHistoryVertexFailedEvent(finalState);
            } catch (IOException e) {
                LOG.error("Failed to send vertex finished event to recovery", e);
            }
            break;
        case KILLED:
        case FAILED:
            addDiagnostic("Vertex " + logIdentifier + " killed/failed due to:" + terminationCause);
            if (!StringUtils.isEmpty(diag)) {
                addDiagnostic(diag);
            }
            abortVertex(VertexStatus.State.valueOf(finalState.name()));
            eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState, terminationCause));
            try {
                logJobHistoryVertexFailedEvent(finalState);
            } catch (IOException e) {
                LOG.error("Failed to send vertex finished event to recovery", e);
            }
            break;
        case SUCCEEDED:
            try {
                try {
                    logJobHistoryVertexFinishedEvent();
                    eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState));
                } catch (LimitExceededException e) {
                    LOG.error("Counter limits exceeded for vertex: " + getLogIdentifier(), e);
                    finalState = VertexState.FAILED;
                    addDiagnostic("Counters limit exceeded: " + e.getMessage());
                    trySetTerminationCause(VertexTerminationCause.COUNTER_LIMITS_EXCEEDED);
                    logJobHistoryVertexFailedEvent(finalState);
                    eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState));
                }
            } catch (IOException e) {
                LOG.error("Failed to send vertex finished event to recovery", e);
                finalState = VertexState.FAILED;
                trySetTerminationCause(VertexTerminationCause.INTERNAL_ERROR);
                eventHandler.handle(new DAGEventVertexCompleted(getVertexId(), finalState));
            }
            break;
        default:
            throw new TezUncheckedException("Unexpected VertexState: " + finalState);
    }
    return finalState;
}