Examples with VertexEventRouteEvent org.apache.tez.dag.app.dag.event.VertexEventRouteEvent used on opensource projects

Example 11 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testVMEventBeforeVertexInitialized.

@Test(timeout = 5000)
public void testVMEventBeforeVertexInitialized() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithCountingVM();
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    VertexImpl v3 = vertices.get("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v1.getState());
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(VertexState.NEW, v3.getState());
    // Generate a VM event for v1, targeted at v3
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    TezEvent tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 1)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(1, v3.pendingVmEvents.size());
    assertEquals(0, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Initialize v2, which will trigger initialization of v3
    dispatcher.getEventHandler().handle(new VertexEvent(v2.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v3.getState());
    // The VM event should have been processed.
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(1, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Send another VM event - make sure it's processed without additional events.
    vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 2)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(2, InvocationCountingVertexManager.numVmEventsReceived.get());
}

Example 12 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexImpl method testInputInitializerEventNoDirectConnection.

@Test(timeout = 10000)
public void testInputInitializerEventNoDirectConnection() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    // Genrate events from v1 to v3's InputInitializer
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", null);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    Assert.assertTrue(initializer.eventReceived.get());
    Assert.assertEquals(3, initializer.stateUpdates.size());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, initializer.stateUpdates.get(0).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING, initializer.stateUpdates.get(1).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED, initializer.stateUpdates.get(2).getVertexState());
}

Example 13 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestVertexManager method testSendCustomProcessorEvent.

@Test(timeout = 5000)
public void testSendCustomProcessorEvent() throws Exception {
    VertexManager vm = new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, mock(StateChangeNotifier.class));
    ArgumentCaptor<VertexEventRouteEvent> requestCaptor = ArgumentCaptor.forClass(VertexEventRouteEvent.class);
    when(mockVertex.getTotalTasks()).thenReturn(2);
    List<CustomProcessorEvent> events = new ArrayList<>();
    // task id too small, should fail
    try {
        vm.pluginContext.sendEventToProcessor(events, -1);
        fail("Should fail for invalid task id");
    } catch (IllegalArgumentException exception) {
        assertTrue(exception.getMessage().contains("Invalid taskId"));
    }
    // task id too large, should fail
    try {
        vm.pluginContext.sendEventToProcessor(events, 10);
        fail("Should fail for invalid task id");
    } catch (IllegalArgumentException exception) {
        assertTrue(exception.getMessage().contains("Invalid taskId"));
    }
    // null event, do nothing
    vm.pluginContext.sendEventToProcessor(null, 0);
    verify(mockHandler, never()).handle(requestCaptor.capture());
    // empty event
    vm.pluginContext.sendEventToProcessor(events, 1);
    verify(mockHandler, never()).handle(requestCaptor.capture());
    // events.add();
    byte[] payload = new byte[] { 1, 2, 3 };
    events.add(CustomProcessorEvent.create(ByteBuffer.wrap(payload)));
    vm.pluginContext.sendEventToProcessor(events, 1);
    verify(mockHandler, times(1)).handle(requestCaptor.capture());
    CustomProcessorEvent cpe = (CustomProcessorEvent) (requestCaptor.getValue().getEvents().get(0).getEvent());
    // should be able to get payload any times
    for (int i = 0; i < 2; i++) {
        ByteBuffer payloadBuffer = cpe.getPayload();
        assertEquals(payload.length, payloadBuffer.remaining());
        for (byte aPayload : payload) {
            assertEquals(aPayload, payloadBuffer.get());
        }
    }
}

Example 14 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting() {
    // Remove after legacy routing is removed
    setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    InputFailedEvent ifEvent = InputFailedEvent.create(0, 1);
    TezEvent tezEvent = new TezEvent(ifEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name()));
}

Example 15 with VertexEventRouteEvent

use of org.apache.tez.dag.app.dag.event.VertexEventRouteEvent in project tez by apache.

the class TestCommit method testVertexRouteEventErrorWhileCommitting.

@Test(timeout = 5000)
public void testVertexRouteEventErrorWhileCommitting() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    // reschedule task
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0);
    TezEvent tezEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
    Assert.assertTrue(v1.commitFutures.isEmpty());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}