Examples with DAGEvent org.apache.tez.dag.app.dag.event.DAGEvent used on opensource projects

Example 1 with DAGEvent

use of org.apache.tez.dag.app.dag.event.DAGEvent in project tez by apache.

the class DAGImpl method handle.

@Override
public /**
 * The only entry point to change the DAG.
 */
void handle(DAGEvent event) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Processing DAGEvent " + event.getDAGId() + " of type " + event.getType() + " while in state " + getInternalState() + ". Event: " + event);
    }
    try {
        writeLock.lock();
        DAGState oldState = getInternalState();
        try {
            getStateMachine().doTransition(event.getType(), event);
        } catch (InvalidStateTransitonException e) {
            String message = "Invalid event " + event.getType() + " on Dag " + this.dagId + " at currentState=" + oldState;
            LOG.error("Can't handle " + message, e);
            addDiagnostic(message);
            eventHandler.handle(new DAGEvent(this.dagId, DAGEventType.INTERNAL_ERROR));
        } catch (RuntimeException e) {
            String message = "Uncaught Exception when handling event " + event.getType() + " on Dag " + this.dagId + " at currentState=" + oldState;
            LOG.error(message, e);
            addDiagnostic(message);
            if (!internalErrorTriggered.getAndSet(true)) {
                // to prevent a recursive loop
                eventHandler.handle(new DAGEvent(this.dagId, DAGEventType.INTERNAL_ERROR));
            }
        }
        // notify the eventhandler of state change
        if (oldState != getInternalState()) {
            LOG.info(dagId + " transitioned from " + oldState + " to " + getInternalState() + " due to event " + event.getType());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 2 with DAGEvent

use of org.apache.tez.dag.app.dag.event.DAGEvent in project tez by apache.

the class TestCommit method testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess.

@Test(timeout = 5000)
public void testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    dag.handle(new DAGEvent(dag.getID(), DAGEventType.INTERNAL_ERROR));
    waitUntil(dag, DAGState.ERROR);
    Assert.assertEquals(DAGTerminationCause.INTERNAL_ERROR, dag.getTerminationCause());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 3 with DAGEvent

use of org.apache.tez.dag.app.dag.event.DAGEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting() {
    // Remove after legacy routing is removed
    setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    InputFailedEvent ifEvent = InputFailedEvent.create(0, 1);
    TezEvent tezEvent = new TezEvent(ifEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name()));
}

Example 4 with DAGEvent

use of org.apache.tez.dag.app.dag.event.DAGEvent in project tez by apache.

the class TestDAGImpl method testDAGInitFailedDuetoInvalidResource.

@Test(timeout = 5000)
public void testDAGInitFailedDuetoInvalidResource() {
    // cluster maxContainerCapability is less than the vertex resource request
    ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(512, 10));
    doReturn(clusterInfo).when(appContext).getClusterInfo();
    dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_INIT));
    dispatcher.await();
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.INIT_FAILURE, dag.getTerminationCause());
    Assert.assertTrue(StringUtils.join(dag.getDiagnostics(), ",").contains("Vertex's TaskResource is beyond the cluster container capability"));
}

Example 5 with DAGEvent

use of org.apache.tez.dag.app.dag.event.DAGEvent in project tez by apache.

the class TestDAGImpl method testNonExistDAGScheduler.

@Test(timeout = 5000)
public void testNonExistDAGScheduler() {
    conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, "non-exist-dag-scheduler");
    dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext);
    dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
    doReturn(dag).when(appContext).getCurrentDAG();
    dag.handle(new DAGEvent(dag.getID(), DAGEventType.DAG_INIT));
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.INIT_FAILURE, dag.getTerminationCause());
    Assert.assertTrue(StringUtils.join(dag.getDiagnostics(), "").contains("java.lang.ClassNotFoundException: non-exist-dag-scheduler"));
}