use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.
the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.
@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
initAllVertices(VertexState.INITED);
VertexImpl v1 = vertices.get("vertex1");
startVertex(v1);
VertexImpl v2 = vertices.get("vertex2");
startVertex(v2);
VertexImpl v3 = vertices.get("vertex3");
VertexImpl v4 = vertices.get("vertex4");
List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
// scheduling start to trigger edge routing to begin
for (int i = 0; i < v4.getTotalTasks(); ++i) {
taskList.add(ScheduleTaskRequest.create(i, null));
}
v4.scheduleTasks(taskList);
Assert.assertEquals(VertexState.RUNNING, v4.getState());
Assert.assertEquals(1, v4.sourceVertices.size());
Edge e = v4.sourceVertices.get(v3);
TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
for (int i = 0; i < 11; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// verify all events have been are in taskEvents
Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
TaskAttemptEventInfo eventInfo;
EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
e.edgeManager = mockPlugin;
when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
// send an input failed event
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
// ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
int fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals(12, fromEventId);
Assert.assertEquals(1, eventInfo.getEvents().size());
Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
// Let failed task send more event
for (int i = 11; i < 14; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// 11 events + 1 INPUT_FAILED_EVENT.
// Events sent out later by failed tasks should not be available.
Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}
use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.
the class TestVertexImpl method testRouteEvent_RecoveredEvent.
@Test(timeout = 5000)
public void testRouteEvent_RecoveredEvent() throws IOException {
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(true).when(appContext).isRecoveryEnabled();
initAllVertices(VertexState.INITED);
VertexImpl v1 = (VertexImpl) vertices.get("vertex1");
VertexImpl v2 = (VertexImpl) vertices.get("vertex2");
VertexImpl v3 = (VertexImpl) vertices.get("vertex3");
startVertex(v1);
startVertex(v2);
TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0);
v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
DataMovementEvent dmEvent = DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0]));
TezTaskAttemptID taId = TezTaskAttemptID.getInstance(taskId, 0);
TezEvent tezEvent1 = new TezEvent(dmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", taId));
v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent1)));
dispatcher.await();
assertTrue(v3.pendingTaskEvents.size() != 0);
// ArgumentCaptor<DAGHistoryEvent> argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
v3.scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null)));
dispatcher.await();
assertTrue(v3.pendingTaskEvents.size() == 0);
// recovery events is not only handled one time
// argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
}
use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.
the class TestVertexImpl method sendTaskGeneratedEvent.
private void sendTaskGeneratedEvent(Event event, EventProducerConsumerType generator, Vertex taskVertex, Vertex edgeVertex) {
TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(taskVertex.getVertexId(), 1), 1);
EventMetaData sourceInfo = new EventMetaData(generator, taskVertex.getName(), edgeVertex.getName(), taId);
sendVertexEventRouteEvent(taskVertex, new TezEvent(event, sourceInfo));
}
use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.
the class TestVertexImpl method testVertexPendingTaskEvents.
@Test(timeout = 5000)
public void testVertexPendingTaskEvents() {
// Remove after bulk routing API is removed
initAllVertices(VertexState.INITED);
VertexImpl v3 = vertices.get("vertex3");
VertexImpl v2 = vertices.get("vertex2");
VertexImpl v1 = vertices.get("vertex1");
startVertex(v1);
TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0);
TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0);
List<TezEvent> taskEvents = Lists.newLinkedList();
TezEvent tezEvent1 = new TezEvent(CompositeDataMovementEvent.create(0, 1, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
TezEvent tezEvent2 = new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
taskEvents.add(tezEvent1);
taskEvents.add(tezEvent2);
// send events and test that they are buffered until some task is scheduled
dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
dispatcher.await();
Assert.assertEquals(2, v3.pendingTaskEvents.size());
v3.scheduleTasks(Collections.singletonList(ScheduleTaskRequest.create(0, null)));
dispatcher.await();
Assert.assertEquals(0, v3.pendingTaskEvents.size());
// send events and test that they are not buffered anymore
dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v3.getVertexId(), taskEvents));
dispatcher.await();
Assert.assertEquals(0, v3.pendingTaskEvents.size());
}
use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.
the class TestVertexImpl method testInputInitializerEvents.
@Test(timeout = 10000)
public void testInputInitializerEvents() throws Exception {
useCustomInitializer = true;
customInitializer = new EventHandlingRootInputInitializer(null);
EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
setupPreDagCreation();
dagPlan = createDAGPlanWithRunningInitializer();
setupPostDagCreation();
VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
initVertex(v1);
startVertex(v1);
Assert.assertEquals(VertexState.RUNNING, v1.getState());
Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
dispatcher.await();
// Wait for the initializer to be invoked - which may be a separate thread.
while (!initializer.initStarted.get()) {
Thread.sleep(10);
}
Assert.assertFalse(initializer.eventReceived.get());
Assert.assertFalse(initializer.initComplete.get());
// Signal the initializer by sending an event - via vertex1
InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null);
// Create taskId and taskAttemptId for the single task that exists in vertex1
TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex2", ta0_t0_v1));
dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
dispatcher.await();
// Events should not be cached in the vertex, since the initializer is running
Assert.assertEquals(0, v2.pendingInitializerEvents.size());
// Verify that events are cached
RootInputInitializerManager.InitializerWrapper initializerWrapper = v2.rootInputInitializerManager.getInitializerWrapper("input1");
Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size());
for (TezTaskID taskId : v1.getTasks().keySet()) {
TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0);
v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED));
v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
dispatcher.await();
v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId());
}
// Both happening in separate threads
while (!initializer.eventReceived.get()) {
Thread.sleep(10);
}
while (!initializer.initComplete.get()) {
Thread.sleep(10);
}
// Will eventually go into RUNNING state, via INITED
while (v2.getState() != VertexState.RUNNING) {
Thread.sleep(10);
}
// Verify the events are no longer cached, but attempts are remembered
Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size());
Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
}
Aggregations