Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 6 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testVertexSuccessToFailedAfterTaskScheduler.

@Test(timeout = 5000)
public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
    // For downstream failures
    VertexImpl v = vertices.get("vertex2");
    List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
    outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig().toUserPayload())).build())).setName("output_v2").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
    v.setAdditionalOutputs(outputs);
    initAllVertices(VertexState.INITED);
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(2, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}

Example 7 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testBasicVertexCompletion.

@Test(timeout = 5000)
public void testBasicVertexCompletion() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(1, v.getCompletedTasks());
    Assert.assertTrue((0.5f) == v.getCompletedTaskProgress());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(2, v.getCompletedTasks());
    Assert.assertTrue((1.0f) == v.getCompletedTaskProgress());
    Assert.assertTrue(v.initTimeRequested > 0);
    Assert.assertTrue(v.initedTime > 0);
    Assert.assertTrue(v.startTimeRequested > 0);
    Assert.assertTrue(v.startedTime > 0);
    Assert.assertTrue(v.finishTime > 0);
}

Example 8 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testVertexWithInitializerParallelismSetTo0.

@Test(timeout = 10000)
public void testVertexWithInitializerParallelismSetTo0() throws InterruptedException, TezException {
    useCustomInitializer = true;
    customInitializer = new RootInitializerSettingParallelismTo0(null);
    RootInitializerSettingParallelismTo0 initializer = (RootInitializerSettingParallelismTo0) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithInitializer0Tasks(RootInitializerSettingParallelismTo0.class.getName());
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    initVertex(v2);
    TezTaskID v2t1 = TezTaskID.getInstance(v2.getVertexId(), 0);
    TezTaskAttemptID ta1V2T1 = TezTaskAttemptID.getInstance(v2t1, 0);
    TezEvent tezEvent = new TezEvent(DataMovementEvent.create(null), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex1", ta1V2T1));
    List<TezEvent> events = new LinkedList<TezEvent>();
    events.add(tezEvent);
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), events));
    startVertex(v2);
    dispatcher.await();
    v2.handle(new VertexEventTaskAttemptCompleted(ta1V2T1, TaskAttemptStateInternal.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v2.getState());
    while (v1.getState() == VertexState.INITIALIZING || v1.getState() == VertexState.INITED) {
        // initializer thread may not have started, so call initializer.go() in the loop all the time
        initializer.go();
        Thread.sleep(10);
    }
    while (v1.getState() != VertexState.SUCCEEDED) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
}

Example 9 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testKilledTasksHandling.

@Test(timeout = 5000)
public void testKilledTasksHandling() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(VertexTerminationCause.OWN_TASK_FAILURE, v.getTerminationCause());
    Assert.assertEquals(TaskState.KILLED, v.getTask(t2).getState());
}

Example 10 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testInputInitializerEventNoDirectConnection.

@Test(timeout = 10000)
public void testInputInitializerEventNoDirectConnection() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer4();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    VertexImplWithRunningInputInitializer v3 = (VertexImplWithRunningInputInitializer) vertices.get("vertex3");
    initVertex(v1);
    startVertex(v1);
    dispatcher.await();
    // Vertex1 start should trigger downstream vertices
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.RUNNING, v2.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
    // Genrate events from v1 to v3's InputInitializer
    InputInitializerEvent event = InputInitializerEvent.create("vertex3", "input1", null);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // Events should not be cached in the vertex, since the initializer is running
    Assert.assertEquals(0, v3.pendingInitializerEvents.size());
    // Events should be cached since the tasks have not succeeded.
    // Verify that events are cached
    RootInputInitializerManager.InitializerWrapper initializerWrapper = v3.rootInputInitializerManager.getInitializerWrapper("input1");
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    // Get all tasks of vertex1 to succeed.
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
        v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
        dispatcher.await();
        v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
    }
    dispatcher.await();
    // Initializer would have run, and processed events.
    while (v3.getState() != VertexState.RUNNING) {
        Thread.sleep(10);
    }
    Assert.assertEquals(VertexState.RUNNING, v3.getState());
    Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
    Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
    Assert.assertTrue(initializer.eventReceived.get());
    Assert.assertEquals(3, initializer.stateUpdates.size());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, initializer.stateUpdates.get(0).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING, initializer.stateUpdates.get(1).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED, initializer.stateUpdates.get(2).getVertexState());
}