Examples with VertexEvent org.apache.tez.dag.app.dag.event.VertexEvent used on opensource projects

Example 36 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testTerminatingVertexForVComplete.

@Test(timeout = 5000)
public void testTerminatingVertexForVComplete() throws Exception {
    setupPreDagCreation();
    dagPlan = createSamplerDAGPlan(false);
    setupPostDagCreation();
    VertexImpl vertex = spy(vertices.get("A"));
    initVertex(vertex);
    startVertex(vertex);
    vertex.handle(new VertexEventTermination(vertex.getVertexId(), VertexTerminationCause.INTERNAL_ERROR));
    vertex.handle(new VertexEvent(vertex.getVertexId(), VertexEventType.V_COMPLETED));
    dispatcher.await();
    Assert.assertTrue(vertex.inTerminalState());
    for (String diagnostic : vertex.getDiagnostics()) {
        if (diagnostic.contains("Invalid event")) {
            fail("Unexpected Invalid event transition!");
        }
    }
}

Example 37 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestCommit method testVertexInternalErrorWhileCommiting.

@Test(timeout = 5000)
public void testVertexInternalErrorWhileCommiting() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    // internal error
    v1.handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.ERROR, v1.getState());
    Assert.assertEquals(VertexTerminationCause.INTERNAL_ERROR, v1.getTerminationCause());
    Assert.assertEquals(DAGState.ERROR, dag.getState());
    Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR, dag.getTerminationCause());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    // commit may not have started, so can't verify commitCounter
    // TODO abort it when internal error happens TEZ-2250
    // Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
// commit may not have started, so can't verify commitCounter
// TODO abort it when internal error happens TEZ-2250
// Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}

Example 38 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestDAGImpl method testDAGErrorAbortAllOutputs.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortAllOutputs() {
    // error on a vertex -> dag error -> all outputs aborted.
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // vertex error -> dag error -> abort all outputs
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.ERROR, v.getState());
    Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(1, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}

Example 39 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class VertexImpl method startIfPossible.

void startIfPossible() {
    if (startSignalPending) {
        // Trigger a start event to ensure route events are seen before
        // a start event.
        LOG.info("Triggering start event for vertex: " + logIdentifier + " with distanceFromRoot: " + distanceFromRoot);
        eventHandler.handle(new VertexEvent(vertexId, VertexEventType.V_START));
    }
}

Example 40 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class VertexImpl method startVertex.

private VertexState startVertex() {
    Preconditions.checkState(getState() == VertexState.INITED, "Vertex must be inited " + logIdentifier);
    if (recoveryData != null && recoveryData.isVertexStarted()) {
        VertexStartedEvent vertexStartedEvent = recoveryData.getVertexStartedEvent();
        this.startedTime = vertexStartedEvent.getStartTime();
    } else {
        this.startedTime = clock.getTime();
    }
    try {
        vertexManager.onVertexStarted(getTaskAttemptIdentifiers(dag, pendingReportedSrcCompletions));
    } catch (AMUserCodeException e) {
        String msg = "Exception in " + e.getSource() + ", vertex=" + logIdentifier;
        LOG.error(msg, e);
        addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause()));
        tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, TaskTerminationCause.AM_USERCODE_FAILURE);
        return VertexState.TERMINATING;
    }
    pendingReportedSrcCompletions.clear();
    logJobHistoryVertexStartedEvent();
    // the vertex is fully configured by the time it starts. Always notify completely configured
    // unless the vertex manager has told us that it is going to reconfigure it further.
    // If the vertex was pre-configured then the event would have been sent out earlier. Calling again
    // would be a no-op. If the vertex was not fully configured and waiting for that to complete then
    // we would start immediately after that. Either parallelism updated (now) or IPO changed (future)
    // or vertex added (future). Simplify these cases by sending the event now automatically for the
    // user as if they had invoked the planned()/done() API's.
    maybeSendConfiguredEvent();
    // when we are ready
    if (targetVertices != null) {
        for (Vertex targetVertex : targetVertices.keySet()) {
            eventHandler.handle(new VertexEventSourceVertexStarted(targetVertex.getVertexId(), getVertexId(), distanceFromRoot));
        }
    }
    // If we have no tasks, just transition to vertex completed
    if (this.numTasks == 0) {
        eventHandler.handle(new VertexEvent(this.vertexId, VertexEventType.V_COMPLETED));
    }
    return VertexState.RUNNING;
}