Examples with VertexEventRouteEvent org.apache.tez.dag.app.dag.event.VertexEventRouteEvent used on opensource projects

Example 26 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testInputInitializerEventsAtNew.

@Test(timeout = 10000)
public void testInputInitializerEventsAtNew() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer3();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex2 has not been INITED, so the rest of the vertices should be in state NEW.
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.NEW, v2.getState());
    Assert.assertEquals(VertexState.NEW, v3.getState());
    // Genrate events from v1 to v3's InputInitializer
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", null);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should be cached in the vertex, since the Initializer has not started
    Assert.assertEquals(1, v3.pendingInitializerEvents.size());
    // Get Vertex1 to succeed before Vertex2 is INITED. Contrived case ? This is likely a tiny race.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        TaskImpl task = (TaskImpl) v1.getTask(taskId);
        task.handle(new TaskEvent(taskId, TaskEventType.T_ATTEMPT_LAUNCHED));
        task.handle(new TaskEventTASucceeded(taskAttemptId));
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Events should still be cached in the vertex
    Assert.assertEquals(1, v3.pendingInitializerEvents.size());
    Assert.assertEquals(VertexState.NEW, v3.getState());
    // Move processing along. INIT the remaining root level vertex.
    initVertex(v2);
    startVertex(v2);
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    // Events should have been cleared from the vertex.
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // KK Add checks to validate thte RootInputManager doesn't remember the events either
    Assert.assertTrue(initializer.eventReceived.get());
    Assert.assertEquals(3, initializer.stateUpdates.size());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, initializer.stateUpdates.get(0).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING, initializer.stateUpdates.get(1).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED, initializer.stateUpdates.get(2).getVertexState());
}

Example 27 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testInputInitializerEventMultipleAttempts.

@Test(timeout = 10000)
public void testInputInitializerEventMultipleAttempts() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    ByteBuffer expected;
    // Genrate events from v1 to v3's InputInitializer
    ByteBuffer payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 0);
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 1);
    expected = payload;
    event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskAttemptID ta1_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 1);
    tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta1_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(2, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        // Make attempt 1 of every task succeed
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(1, initializer.initializerEvents.size());
    Assert.assertEquals(expected, initializer.initializerEvents.get(0).getUserPayload());
}

Example 28 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteDataMovementEventToDestination.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteDataMovementEventToDestination() {
    setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0]));
    TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name()));
}

Example 29 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class VertexImpl method initializeVertex.

private boolean initializeVertex() {
    // Don't need to initialize committer if vertex is fully completed
    if (recoveryData != null && recoveryData.shouldSkipInit()) {
        // Do other necessary recovery here
        initedTime = recoveryData.getVertexInitedEvent().getInitedTime();
        List<TezEvent> initGeneratedEvents = recoveryData.getVertexInitedEvent().getInitGeneratedEvents();
        if (initGeneratedEvents != null && !initGeneratedEvents.isEmpty()) {
            eventHandler.handle(new VertexEventRouteEvent(getVertexId(), initGeneratedEvents));
        }
        // reset rootInputDescriptor because it may be changed during input initialization.
        this.rootInputDescriptors = recoveryData.getVertexInitedEvent().getAdditionalInputs();
    } else {
        initedTime = clock.getTime();
    }
    // state in recovery mode
    if (recoveryData == null || recoveryData.getVertexFinishedEvent() == null) {
        try {
            initializeCommitters();
        } catch (Exception e) {
            LOG.warn("Vertex Committer init failed, vertex=" + logIdentifier, e);
            addDiagnostic("Vertex init failed : " + ExceptionUtils.getStackTrace(e));
            trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
            finished(VertexState.FAILED);
            return false;
        }
    }
    logJobHistoryVertexInitializedEvent();
    return true;
}

Example 30 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TaskAttemptImpl method sendInputFailedToConsumers.

@VisibleForTesting
protected void sendInputFailedToConsumers() {
    Vertex vertex = getVertex();
    Map<Vertex, Edge> edges = vertex.getOutputVertices();
    if (edges != null && !edges.isEmpty()) {
        List<TezEvent> tezIfEvents = Lists.newArrayListWithCapacity(edges.size());
        for (Vertex edgeVertex : edges.keySet()) {
            tezIfEvents.add(new TezEvent(new InputFailedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, vertex.getName(), edgeVertex.getName(), getID()), appContext.getClock().getTime()));
        }
        sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents));
    }
}