Examples with VertexEventRouteEvent org.apache.tez.dag.app.dag.event.VertexEventRouteEvent used on opensource projects

Example 21 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testInputInitializerEventsMultipleSources.

@Test(timeout = 10000)
public void testInputInitializerEventsMultipleSources() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    initializer.setNumExpectedEvents(4);
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    List<ByteBuffer> expectedPayloads = new LinkedList<ByteBuffer>();
    // Genrate events from v1 to v3's InputInitializer
    ByteBuffer payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 0);
    expectedPayloads.add(payload);
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    Assert.assertEquals(1, initializer.initializerEvents.size());
    // Test written based on this
    Assert.assertEquals(2, v2.getTotalTasks());
    // Generate events from v2 to v3's initializer. 1 from task 0, 2 from task 1
    for (Task task : v2.getTasks().values()) {
        TezTaskID taskId = task.getTaskId();
        TezTaskAttemptID attemptId = TezTaskAttemptID.getInstance(taskId, 0);
        int numEventsFromTask = taskId.getId() + 1;
        for (int i = 0; i < numEventsFromTask; i++) {
            payload = ByteBuffer.allocate(12).putInt(0, 2).putInt(4, taskId.getId()).putInt(8, i);
            expectedPayloads.add(payload);
            InputInitializerEvent event2 = InputInitializerEvent.create("vertex3", "input1", payload);
            TezEvent tezEvent2 = new TezEvent(event2, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", attemptId));
            dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Collections.singletonList(tezEvent2)));
            dispatcher.await();
        }
    }
    // Validate queueing of these events
    // Only v2 events pending
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().keySet().size());
    // 3 events pending
    Assert.assertEquals(3, initializerWrapper.getPendingEvents().get(v2.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v2.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v2.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v2.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v2.stateChangeNotifier.taskSucceeded(v2.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(4, initializer.initializerEvents.size());
    Assert.assertTrue(initializer.initComplete.get());
    Assert.assertEquals(2, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    for (InputInitializerEvent initializerEvent : initializer.initializerEvents) {
        expectedPayloads.remove(initializerEvent.getUserPayload());
    }
    Assert.assertEquals(0, expectedPayloads.size());
}

Example 22 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
    // Let failed task send more event
    for (int i = 11; i < 14; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // 11 events + 1 INPUT_FAILED_EVENT.
    // Events sent out later by failed tasks should not be available.
    Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 23 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testRouteEvent_RecoveredEvent.

@Test(timeout = 5000)
public void testRouteEvent_RecoveredEvent() throws IOException {
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(true).when(appContext).isRecoveryEnabled();
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = (VertexImpl) vertices.get("vertex1");
    VertexImpl v2 = (VertexImpl) vertices.get("vertex2");
    VertexImpl v3 = (VertexImpl) vertices.get("vertex3");
    startVertex(v1);
    startVertex(v2);
    TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0);
    v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
    DataMovementEvent dmEvent = DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(taskId, 0);
    TezEvent tezEvent1 = new TezEvent(dmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent1)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() != 0);
    // ArgumentCaptor<DAGHistoryEvent> argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    // verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
    // verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
    v3.scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() == 0);
// recovery events is not only handled one time
// argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
}

Example 24 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexPendingTaskEvents.

@Test(timeout = 5000)
public void testVertexPendingTaskEvents() {
    // Remove after bulk routing API is removed
    initAllVertices(VertexState.INITED);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v2 = vertices.get("vertex2");
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0);
    List<TezEvent> taskEvents = Lists.newLinkedList();
    TezEvent tezEvent1 = new TezEvent(CompositeDataMovementEvent.create(0, 1, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    TezEvent tezEvent2 = new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    taskEvents.add(tezEvent1);
    taskEvents.add(tezEvent2);
    // send events and test that they are buffered until some task is scheduled
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(2, v3.pendingTaskEvents.size());
    v3.scheduleTasks(Collections.singletonList(ScheduleTaskRequest.create(0, null)));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
    // send events and test that they are not buffered anymore
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
    dispatcher.await();
    Assert.assertEquals(0, v3.pendingTaskEvents.size());
}

Example 25 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testInputInitializerEvents.

@Test(timeout = 10000)
public void testInputInitializerEvents() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    initVertex(v1);
    startVertex(v1);
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    dispatcher.await();
    // Wait for the initializer to be invoked - which may be a separate thread.
    while (!initializer.initStarted.get()) {
        Thread.sleep(10);
    }
    Assert.assertFalse(initializer.eventReceived.get());
    Assert.assertFalse(initializer.initComplete.get());
    // Signal the initializer by sending an event - via vertex1
    InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex2", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v2.pendingInitializerEvents.size());
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v2.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    // Both happening in separate threads
    while (!initializer.eventReceived.get()) {
        Thread.sleep(10);
    }
    while (!initializer.initComplete.get()) {
        Thread.sleep(10);
    }
    // Will eventually go into RUNNING state, via INITED
    while (v2.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    // Verify the events are no longer cached, but attempts are remembered
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
}