Examples with VertexEvent org.apache.tez.dag.app.dag.event.VertexEvent used on opensource projects

Example 11 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testTaskSchedulingWithCustomEdges.

@Test(timeout = 5000)
public /**
 * Ref: TEZ-1494
 * If broadcast, one-to-one or custom edges are present in source, tasks should not start until
 * 1 task from each source vertex is complete.
 */
void testTaskSchedulingWithCustomEdges() throws TezException {
    setupPreDagCreation();
    dagPlan = createCustomDAGWithCustomEdges();
    setupPostDagCreation();
    /**
     *   M2 --(SG)--> R3 --(B)--\
     *                           \
     *   M7 --(B)---------------->M5 ---(SG)--> R6
     *                            /
     *   M8---(C)--------------->/
     *                          /
     *   M9---(B)--------------> (zero task vertex)
     */
    // init M2
    VertexImpl m2 = vertices.get("M2");
    VertexImpl m7 = vertices.get("M7");
    VertexImpl r3 = vertices.get("R3");
    VertexImpl m5 = vertices.get("M5");
    VertexImpl m8 = vertices.get("M8");
    VertexImpl m9 = vertices.get("M9");
    initVertex(m2);
    initVertex(m7);
    initVertex(m8);
    initVertex(m9);
    assertTrue(m7.getState().equals(VertexState.INITED));
    assertTrue(m9.getState().equals(VertexState.INITED));
    assertTrue(m5.getState().equals(VertexState.INITED));
    assertTrue(m8.getState().equals(VertexState.INITED));
    assertTrue(m5.getVertexManager().getPlugin() instanceof ImmediateStartVertexManager);
    // Start M9
    dispatcher.getEventHandler().handle(new VertexEvent(m9.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // Start M2; Let tasks complete in M2; Also let 1 task complete in R3
    dispatcher.getEventHandler().handle(new VertexEvent(m2.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // R3 should be in running state
    assertTrue(r3.getState().equals(VertexState.RUNNING));
    // Let us start M7; M5 should start not start as it is dependent on M8 as well
    dispatcher.getEventHandler().handle(new VertexEvent(m7.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // M5 should be in INITED state, as it depends on M8
    assertTrue(m5.getState().equals(VertexState.INITED));
    for (Task task : m5.getTasks().values()) {
        assertTrue(task.getState().equals(TaskState.NEW));
    }
    // Let us start M8; M5 should start now
    dispatcher.getEventHandler().handle(new VertexEvent(m8.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    assertTrue(m9.getState().equals(VertexState.SUCCEEDED));
    // M5 in running state. All source vertices have started and are configured
    assertTrue(m5.getState().equals(VertexState.RUNNING));
    for (Task task : m5.getTasks().values()) {
        assertTrue(task.getState().equals(TaskState.SCHEDULED));
    }
}

Example 12 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testExceptionFromVM_OnVertexManagerVertexStateUpdated.

@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithVMException("TestVMStateUpdate", VMExceptionLocation.OnVertexManagerVertexStateUpdated);
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    // wait to ensure init is completed, so that rootInitManager is not null
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    startVertex(v1, false);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
    assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerVertexStateUpdated.name()));
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause());
}

Example 13 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccessLegacyRouting.

@Test(timeout = 5000)
public void testVertexWithInitializerSuccessLegacyRouting() throws Exception {
    // Remove after legacy routing is removed
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    // task events get buffered
    Assert.assertEquals(5, v1.pendingTaskEvents.size());
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    Assert.assertEquals(1, v2.pendingTaskEvents.size());
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // task events get buffered
    Assert.assertEquals(11, v2.pendingTaskEvents.size());
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 14 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method initVertex.

private void initVertex(VertexImpl v) {
    Assert.assertEquals(VertexState.NEW, v.getState());
    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
}

Example 15 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerFailure.

@Test(timeout = 5000)
public void testVertexWithInitializerFailure() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    initializerManager1.failInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",").contains(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE.name()));
    Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",").contains("MockInitializerFailed"));
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    initializerManager2.failInputInitialization();
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",").contains(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE.name()));
    Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",").contains("MockInitializerFailed"));
}