Examples with VertexEvent org.apache.tez.dag.app.dag.event.VertexEvent used on opensource projects

Example 16 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerSuccess.

@Test(timeout = 500000)
public void testVertexWithInitializerSuccess() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInputInitializer("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(5, v1.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v1.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v1Hints.size(); ++i) {
        Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    for (int i = 0; i < 5; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v1.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v1.getTotalTasks(); ++i) {
        Assert.assertEquals(1, v1.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v1.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2");
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    // non-task events don't get buffered
    List<TezEvent> events = Lists.newLinkedList();
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    events.add(new TezEvent(VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", ta0_t0_v1)));
    events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "NULL_VERTEX", null)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), events));
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager();
    List<TaskLocationHint> v2Hints = createTaskLocationHints(10);
    initializerManager2.completeInputInitialization(0, 10, v2Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v2.getState());
    Assert.assertEquals(10, v2.getTotalTasks());
    Assert.assertEquals(RootInputVertexManager.class.getName(), v2.getVertexManager().getPlugin().getClass().getName());
    for (int i = 0; i < v2Hints.size(); ++i) {
        Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]);
    }
    Assert.assertEquals(true, initializerManager2.hasShutDown);
    // scheduling start to trigger edge routing to begin
    taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v2.scheduleTasks(taskList);
    dispatcher.await();
    // check all tasks get their events
    for (int i = 0; i < v2.getTotalTasks(); ++i) {
        Assert.assertEquals(((i == 0) ? 2 : 1), v2.getTaskAttemptTezEvents(TezTaskAttemptID.getInstance(v2.getTask(i).getTaskId(), 0), 0, 0, 100).getEvents().size());
    }
    for (int i = 0; i < 10; i++) {
        List<InputSpec> inputSpecs = v1.getInputSpecList(i);
        Assert.assertEquals(1, inputSpecs.size());
        Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount());
    }
}

Example 17 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVMEventBeforeVertexInitialized.

@Test(timeout = 5000)
public void testVMEventBeforeVertexInitialized() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithCountingVM();
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    VertexImpl v3 = vertices.get("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v1.getState());
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(VertexState.NEW, v3.getState());
    // Generate a VM event for v1, targeted at v3
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    TezEvent tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 1)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(1, v3.pendingVmEvents.size());
    assertEquals(0, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Initialize v2, which will trigger initialization of v3
    dispatcher.getEventHandler().handle(new VertexEvent(v2.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v3.getState());
    // The VM event should have been processed.
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(1, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Send another VM event - make sure it's processed without additional events.
    vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 2)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(2, InvocationCountingVertexManager.numVmEventsReceived.get());
}

Example 18 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexNoTasksTerminated.

@Test(timeout = 10000)
public void testVertexNoTasksTerminated() {
    TezVertexID vId = null;
    try {
        TezDAGID invalidDagId = TezDAGID.getInstance(dagId.getApplicationId(), 1000);
        vId = TezVertexID.getInstance(invalidDagId, 1);
        VertexPlan vPlan = invalidDagPlan.getVertex(0);
        EventHandler mockEventHandler = mock(EventHandler.class);
        VertexImpl v = new VertexImpl(vId, vPlan, vPlan.getName(), conf, mockEventHandler, taskCommunicatorManagerInterface, clock, thh, true, appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption, updateTracker, new Configuration(false));
        v.setInputVertices(new HashMap());
        vertexIdMap.put(vId, v);
        vertices.put(v.getName(), v);
        v.handle(new VertexEvent(vId, VertexEventType.V_INIT));
        Assert.assertEquals(VertexState.INITED, v.getState());
        v.handle(new VertexEvent(vId, VertexEventType.V_START));
        Assert.assertEquals(VertexState.RUNNING, v.getState());
        v.handle(new VertexEventTermination(vId, VertexTerminationCause.OTHER_VERTEX_FAILURE));
        Assert.assertEquals(VertexState.TERMINATING, v.getState());
        v.handle(new VertexEvent(vId, VertexEventType.V_COMPLETED));
        Assert.assertEquals(VertexState.KILLED, v.getState());
        Assert.assertTrue(1.0f == v.getCompletedTaskProgress());
    } finally {
        if (vId != null) {
            vertexIdMap.remove(vId);
        }
    }
}

Example 19 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testStartWithUninitializedCustomEdge.

@Test(timeout = 5000)
public void testStartWithUninitializedCustomEdge() throws Exception {
    // Race when a source vertex manages to start before the target vertex has
    // been initialized
    setupPreDagCreation();
    dagPlan = createSamplerDAGPlan(true);
    setupPostDagCreation();
    VertexImpl vA = vertices.get("A");
    VertexImpl vB = vertices.get("B");
    VertexImpl vC = vertices.get("C");
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT));
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    // setting the edge manager should vA to start
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = vC.sourceVertices.get(vA);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Map<String, EdgeProperty> edges = Maps.newHashMap();
    edges.put("B", edgeProp);
    vC.reconfigureVertex(2, vertexLocationHint, edges);
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.RUNNING, vB.getState());
    Assert.assertEquals(VertexState.RUNNING, vC.getState());
    Assert.assertNotNull(vA.getTask(0));
    Assert.assertNotNull(vB.getTask(0));
    Assert.assertNotNull(vC.getTask(0));
}

Example 20 with VertexEvent

use of org.apache.tez.dag.app.dag.event.VertexEvent in project tez by apache.

the class TestVertexImpl method testVertexWithMultipleInitializers1.

@Test(timeout = 5000)
public void testVertexWithMultipleInitializers1() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithMultipleInitializers("TestInputInitializer");
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    List<TaskLocationHint> v1Hints = createTaskLocationHints(5);
    // Complete initializer which sets parallelism first
    initializerManager1.completeInputInitialization(0, 5, v1Hints);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    Assert.assertEquals(1, v1.numInitializerCompletionsHandled);
    // Complete second initializer
    initializerManager1.completeInputInitialization(1);
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, v1.getState());
    Assert.assertEquals(2, v1.numInitializerCompletionsHandled);
}