Search in sources :

Example 1 with VertexEventTaskReschedule

use of org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule in project tez by apache.

the class TestVertexImpl method testVertexSuccessToRunningAfterTaskScheduler.

@Test(timeout = 5000)
public void testVertexSuccessToRunningAfterTaskScheduler() {
    // For downstream failures
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_RERUNNING).intValue());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(2, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
Also used : VertexEventTaskReschedule(org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule) VertexEventTaskCompleted(org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted) TezTaskID(org.apache.tez.dag.records.TezTaskID) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Example 2 with VertexEventTaskReschedule

use of org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule in project tez by apache.

the class TestVertexImpl method testVertexSuccessToFailedAfterTaskScheduler.

@Test(timeout = 5000)
public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
    // For downstream failures
    VertexImpl v = vertices.get("vertex2");
    List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
    outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig().toUserPayload())).build())).setName("output_v2").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
    v.setAdditionalOutputs(outputs);
    initAllVertices(VertexState.INITED);
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(2, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
Also used : VertexEventTaskReschedule(org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule) RootInputLeafOutputProto(org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto) ArrayList(java.util.ArrayList) VertexEventTaskCompleted(org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted) TezTaskID(org.apache.tez.dag.records.TezTaskID) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Example 3 with VertexEventTaskReschedule

use of org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule in project tez by apache.

the class TestDAGImpl method testVertexReRunning.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexReRunning() {
    initDAG(dag);
    dag.dagScheduler = mock(DAGScheduler.class);
    startDAG(dag);
    dispatcher.await();
    TezVertexID vId = TezVertexID.getInstance(dagId, 1);
    Vertex v = dag.getVertex(vId);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 1), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dag.getSuccessfulVertices());
    Assert.assertEquals(1, dag.numCompletedVertices);
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(TezTaskID.getInstance(vId, 0)));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(0, dag.getSuccessfulVertices());
    Assert.assertEquals(0, dag.numCompletedVertices);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dag.getSuccessfulVertices());
    Assert.assertEquals(1, dag.numCompletedVertices);
}
Also used : Vertex(org.apache.tez.dag.app.dag.Vertex) VertexEventTaskReschedule(org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule) DAGScheduler(org.apache.tez.dag.app.dag.DAGScheduler) VertexEventTaskCompleted(org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted) TezVertexID(org.apache.tez.dag.records.TezVertexID) Test(org.junit.Test) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest)

Example 4 with VertexEventTaskReschedule

use of org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule in project tez by apache.

the class TestVertexImpl method testCompletedStatsCache.

@Test(timeout = 5000)
public void testCompletedStatsCache() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    dispatcher.getEventHandler().handle(new TaskEventTALaunched(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    VertexStatistics stats = v.getStatistics();
    // Ensure that task 0 is available in completed stats cache
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.get(0));
    // Reschedule task 0
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    // cache should be cleared
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.cardinality() == 0);
}
Also used : VertexEventTaskReschedule(org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule) TaskEventTALaunched(org.apache.tez.dag.app.dag.event.TaskEventTALaunched) VertexStatistics(org.apache.tez.runtime.api.VertexStatistics) TaskEventTASucceeded(org.apache.tez.dag.app.dag.event.TaskEventTASucceeded) VertexEventTaskCompleted(org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted) TezTaskID(org.apache.tez.dag.records.TezTaskID) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Example 5 with VertexEventTaskReschedule

use of org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule in project tez by apache.

the class TestVertexImpl method testTaskReschedule.

@Test(timeout = 5000)
public void testTaskReschedule() {
    // For downstream failures
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    // FIXME need to handle dups
    // dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}
Also used : VertexEventTaskReschedule(org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule) VertexEventTaskCompleted(org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted) TezTaskID(org.apache.tez.dag.records.TezTaskID) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Aggregations

VertexEventTaskCompleted (org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted)7 VertexEventTaskReschedule (org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule)7 Test (org.junit.Test)7 StateChangeNotifierForTest (org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest)5 TezTaskID (org.apache.tez.dag.records.TezTaskID)5 EdgeManagerForTest (org.apache.tez.test.EdgeManagerForTest)4 GraceShuffleVertexManagerForTest (org.apache.tez.test.GraceShuffleVertexManagerForTest)4 VertexManagerPluginForTest (org.apache.tez.test.VertexManagerPluginForTest)4 ArrayList (java.util.ArrayList)1 RootInputLeafOutputProto (org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto)1 DAGScheduler (org.apache.tez.dag.app.dag.DAGScheduler)1 Vertex (org.apache.tez.dag.app.dag.Vertex)1 TaskEventTALaunched (org.apache.tez.dag.app.dag.event.TaskEventTALaunched)1 TaskEventTASucceeded (org.apache.tez.dag.app.dag.event.TaskEventTASucceeded)1 TezVertexID (org.apache.tez.dag.records.TezVertexID)1 VertexStatistics (org.apache.tez.runtime.api.VertexStatistics)1