Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 36 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testVertexFailure.

@Test(timeout = 5000)
public void testVertexFailure() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(VertexTerminationCause.OWN_TASK_FAILURE, v.getTerminationCause());
    String diagnostics = StringUtils.join(v.getDiagnostics(), ",").toLowerCase(Locale.ENGLISH);
    assertTrue(diagnostics.contains("task failed" + ", taskid=" + t1.toString()));
}

Example 37 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testTaskFailedAfterVertexSuccess.

@Test(timeout = 5000)
public void testTaskFailedAfterVertexSuccess() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    CountingOutputCommitter committer = (CountingOutputCommitter) v.getOutputCommitter("outputx");
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, committer.commitCounter);
    Assert.assertEquals(0, committer.abortCounter);
    Assert.assertEquals(1, committer.initCounter);
    Assert.assertEquals(1, committer.setupCounter);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(1, committer.commitCounter);
    Assert.assertEquals(1, committer.abortCounter);
}

Example 38 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testInputInitializerEventMultipleAttempts.

@Test(timeout = 10000)
public void testInputInitializerEventMultipleAttempts() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    ByteBuffer expected;
    // Genrate events from v1 to v3's InputInitializer
    ByteBuffer payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 0);
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    payload = ByteBuffer.allocate(12).putInt(0, 1).putInt(4, 0).putInt(8, 1);
    expected = payload;
    event = InputInitializerEvent.create("vertex3", "input1", payload);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskAttemptID ta1_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 1);
    tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta1_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(2, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        // Make attempt 1 of every task succeed
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(1, initializer.initializerEvents.size());
    Assert.assertEquals(expected, initializer.initializerEvents.get(0).getUserPayload());
}

Example 39 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testDAGEventGeneration.

@Test(timeout = 5000)
public void testDAGEventGeneration() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}

Example 40 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testTaskReschedule.

@Test(timeout = 5000)
public void testTaskReschedule() {
    // For downstream failures
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    // FIXME need to handle dups
    // dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}