Examples with VertexEventRouteEvent org.apache.tez.dag.app.dag.event.VertexEventRouteEvent used on opensource projects

Example 6 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testExceptionFromVM_OnVertexManagerEventReceived.

@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithVMException("TestInputInitializer", VMExceptionLocation.OnVertexManagerEventReceived);
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    // wait to ensure init is completed, so that rootInitManager is not null
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    VertexManagerEvent vmEvent = VertexManagerEvent.create(v1.getName(), ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId1 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0);
    TezEvent tezEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, v1.getName(), null, taId1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
    Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name()));
}

Example 7 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccessLegacyRouting.

@Test(timeout = 5000)
public void testVertexWithInitializerSuccessLegacyRouting() throws Exception {
    // Remove after legacy routing is removed
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    // task events get buffered
    Assert.assertEquals(5, v1.pendingTaskEvents.size());
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    Assert.assertEquals(1, v2.pendingTaskEvents.size());
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // task events get buffered
    Assert.assertEquals(11, v2.pendingTaskEvents.size());
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 8 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerParallelismSetTo0.

@Test(timeout = 10000)
public void testVertexWithInitializerParallelismSetTo0() throws InterruptedException, TezException {
    useCustomInitializer = true;
    customInitializer = new RootInitializerSettingParallelismTo0(null);
    RootInitializerSettingParallelismTo0 initializer = (RootInitializerSettingParallelismTo0) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInitializer0Tasks(RootInitializerSettingParallelismTo0.class.getName());
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    initVertex(v2);
    TezTaskID v2t1 = TezTaskID.getInstance(v2.getVertexId(), 0);
    TezTaskAttemptID ta1V2T1 = TezTaskAttemptID.getInstance(v2t1, 0);
    TezEvent tezEvent = new TezEvent(DataMovementEvent.create(null), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex1", ta1V2T1));
    List<TezEvent> events = new LinkedList<TezEvent>();
    events.add(tezEvent);
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), events));
    startVertex(v2);
    dispatcher.await();
    v2.handle(new VertexEventTaskAttemptCompleted(ta1V2T1, TaskAttemptStateInternal.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    while (v1.getState() == VertexState.INITIALIZING || v1.getState() == VertexState.INITED) {
        // initializer thread may not have started, so call initializer.go() in the loop all the time
        initializer.go();
        Thread.sleep(10);
    }
    while (v1.getState() != VertexState.SUCCEEDED) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
}

Example 9 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccess.

@Test(timeout = 500000)
public void testVertexWithInitializerSuccess() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v1.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        Assert.assertEquals(1, v1.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v1.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // scheduling start to trigger edge routing to begin
    taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v2.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        Assert.assertEquals(((i == 0) ? 2 : 1), v2.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v2.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 10 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttempts.

@Test(timeout = 5000)
public void testVertexGetTAAttempts() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 5; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been put in pending.
    // this is not necessary after legacy routing has been removed
    Assert.assertEquals(5, v4.pendingTaskEvents.size());
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    dispatcher.await();
    // verify all events have been moved to taskEvents
    Assert.assertEquals(5, v4.getOnDemandRouteEvents().size());
    for (int i = 5; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    // source task id will not match. all events will return null
    when(mockPlugin.routeDataMovementEventToDestination(1, 0, 0)).thenReturn(mockRoute);
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, 0, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    int fromEventId = 0;
    // source task id will match. all events will be returned
    // max events is respected.
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    for (int i = 0; i < 11; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i + 1), fromEventId);
        Assert.assertEquals(1, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    // ask for events with sufficient buffer. get all events in a single shot.
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(11, fromEventId);
    Assert.assertEquals(11, eventInfo.getEvents().size());
    // change max events to larger value. max events does not evenly divide total events
    fromEventId = 0;
    for (int i = 1; i <= 2; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i * 5), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(1, eventInfo.getEvents().size());
    // return more events that dont evenly fit in max size
    mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    fromEventId = 0;
    int lastFromEventId = 0;
    for (int i = 1; i <= 4; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i % 2 > 0 ? (lastFromEventId += 2) : (lastFromEventId += 3)), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(2, eventInfo.getEvents().size());
}