use of org.apache.tez.dag.app.TaskAttemptEventInfo in project tez by apache.
the class VertexImpl method getTaskAttemptTezEvents.
@Override
public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, int fromEventId, int preRoutedFromEventId, int maxEvents) {
Task task = getTask(attemptID.getTaskID());
ArrayList<TezEvent> events = task.getTaskAttemptTezEvents(attemptID, preRoutedFromEventId, maxEvents);
int nextPreRoutedFromEventId = preRoutedFromEventId + events.size();
int nextFromEventId = fromEventId;
onDemandRouteEventsReadLock.lock();
try {
int currEventCount = onDemandRouteEvents.size();
try {
if (currEventCount > fromEventId) {
if (events != TaskImpl.EMPTY_TASK_ATTEMPT_TEZ_EVENTS) {
events.ensureCapacity(maxEvents);
} else {
events = Lists.newArrayListWithCapacity(maxEvents);
}
int numPreRoutedEvents = events.size();
int taskIndex = attemptID.getTaskID().getId();
Preconditions.checkState(taskIndex < tasks.size(), "Invalid task index for TA: " + attemptID + " vertex: " + getLogIdentifier());
boolean isFirstEvent = true;
boolean firstEventObsoleted = false;
for (nextFromEventId = fromEventId; nextFromEventId < currEventCount; ++nextFromEventId) {
boolean earlyExit = false;
if (events.size() == maxEvents) {
break;
}
EventInfo eventInfo = onDemandRouteEvents.get(nextFromEventId);
if (eventInfo.isObsolete) {
// ignore obsolete events
firstEventObsoleted = true;
continue;
}
TezEvent tezEvent = eventInfo.tezEvent;
switch(tezEvent.getEventType()) {
case INPUT_FAILED_EVENT:
case DATA_MOVEMENT_EVENT:
case COMPOSITE_DATA_MOVEMENT_EVENT:
{
int srcTaskIndex = eventInfo.eventTaskIndex;
Edge srcEdge = eventInfo.eventEdge;
PendingEventRouteMetadata pendingRoute = null;
if (isFirstEvent) {
// the first event is the one that can have pending routes because its expanded
// events had not been completely sent in the last round.
isFirstEvent = false;
pendingRoute = srcEdge.removePendingEvents(attemptID);
if (pendingRoute != null) {
// obsoleted
if (tezEvent != pendingRoute.getTezEvent()) {
Preconditions.checkState(firstEventObsoleted);
// pending routes can be ignored for obsoleted events
pendingRoute = null;
}
}
}
if (!srcEdge.maybeAddTezEventForDestinationTask(tezEvent, attemptID, srcTaskIndex, events, maxEvents, pendingRoute)) {
// not enough space left for this iteration events.
// Exit and start from here next time
earlyExit = true;
}
}
break;
case ROOT_INPUT_DATA_INFORMATION_EVENT:
{
InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent();
if (riEvent.getTargetIndex() == taskIndex) {
events.add(tezEvent);
}
}
break;
default:
throw new TezUncheckedException("Unexpected event type for task: " + tezEvent.getEventType());
}
if (earlyExit) {
break;
}
}
int numEventsSent = events.size() - numPreRoutedEvents;
if (numEventsSent > 0) {
StringBuilder builder = new StringBuilder();
builder.append("Sending ").append(attemptID).append(" ").append(numEventsSent).append(" events [").append(fromEventId).append(",").append(nextFromEventId).append(") total ").append(currEventCount).append(" ").append(getLogIdentifier());
LOG.info(builder.toString());
}
}
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() + ", vertex=" + getLogIdentifier();
LOG.error(msg, e);
eventHandler.handle(new VertexEventManagerUserCodeError(getVertexId(), e));
nextFromEventId = fromEventId;
events.clear();
}
} finally {
onDemandRouteEventsReadLock.unlock();
}
if (!events.isEmpty()) {
for (int i = (events.size() - 1); i >= 0; --i) {
TezEvent lastEvent = events.get(i);
// record the last event sent by the AM to the task
EventType lastEventType = lastEvent.getEventType();
// if the following changes then critical path logic/recording may need revision
if (lastEventType == EventType.COMPOSITE_DATA_MOVEMENT_EVENT || lastEventType == EventType.COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT || lastEventType == EventType.DATA_MOVEMENT_EVENT || lastEventType == EventType.ROOT_INPUT_DATA_INFORMATION_EVENT) {
task.getAttempt(attemptID).setLastEventSent(lastEvent);
break;
}
}
}
return new TaskAttemptEventInfo(nextFromEventId, events, nextPreRoutedFromEventId);
}
use of org.apache.tez.dag.app.TaskAttemptEventInfo in project tez by apache.
the class TestVertexImpl method testVertexGetTAAttemptsObsoletionWithPendingRoutes.
@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletionWithPendingRoutes() throws Exception {
initAllVertices(VertexState.INITED);
VertexImpl v1 = vertices.get("vertex1");
startVertex(v1);
VertexImpl v2 = vertices.get("vertex2");
startVertex(v2);
VertexImpl v3 = vertices.get("vertex3");
VertexImpl v4 = vertices.get("vertex4");
List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
// scheduling start to trigger edge routing to begin
for (int i = 0; i < v4.getTotalTasks(); ++i) {
taskList.add(ScheduleTaskRequest.create(i, null));
}
v4.scheduleTasks(taskList);
Assert.assertEquals(VertexState.RUNNING, v4.getState());
Assert.assertEquals(1, v4.sourceVertices.size());
Edge e = v4.sourceVertices.get(v3);
TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
for (int i = 0; i < 11; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// verify all events have been are in taskEvents
Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
TaskAttemptEventInfo eventInfo;
EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
EventRouteMetadata mockFailedRoute = EventRouteMetadata.create(1, new int[] { 0 });
e.edgeManager = mockPlugin;
when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockFailedRoute);
// return more events that dont evenly fit in max size
EventRouteMetadata mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
int fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
fromEventId = eventInfo.getNextFromEventId();
// 0-1 events expanded and fit, 2nd event has pending routes
Assert.assertEquals(2, fromEventId);
Assert.assertEquals(5, eventInfo.getEvents().size());
// send an input failed event
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
// get only input failed event. all DM events obsoleted
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals(12, fromEventId);
Assert.assertEquals(1, eventInfo.getEvents().size());
Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}
use of org.apache.tez.dag.app.TaskAttemptEventInfo in project tez by apache.
the class TestVertexImpl method testVertexGetTAAttempts.
@Test(timeout = 5000)
public void testVertexGetTAAttempts() throws Exception {
initAllVertices(VertexState.INITED);
VertexImpl v1 = vertices.get("vertex1");
startVertex(v1);
VertexImpl v2 = vertices.get("vertex2");
startVertex(v2);
VertexImpl v3 = vertices.get("vertex3");
VertexImpl v4 = vertices.get("vertex4");
Assert.assertEquals(VertexState.RUNNING, v4.getState());
Assert.assertEquals(1, v4.sourceVertices.size());
Edge e = v4.sourceVertices.get(v3);
TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
for (int i = 0; i < 5; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// verify all events have been put in pending.
// this is not necessary after legacy routing has been removed
Assert.assertEquals(5, v4.pendingTaskEvents.size());
List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
// scheduling start to trigger edge routing to begin
for (int i = 0; i < v4.getTotalTasks(); ++i) {
taskList.add(ScheduleTaskRequest.create(i, null));
}
v4.scheduleTasks(taskList);
dispatcher.await();
// verify all events have been moved to taskEvents
Assert.assertEquals(5, v4.getOnDemandRouteEvents().size());
for (int i = 5; i < 11; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// verify all events have been are in taskEvents
Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
TaskAttemptEventInfo eventInfo;
EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
e.edgeManager = mockPlugin;
// source task id will not match. all events will return null
when(mockPlugin.routeDataMovementEventToDestination(1, 0, 0)).thenReturn(mockRoute);
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, 0, 0, 1);
// all events traversed
Assert.assertEquals(11, eventInfo.getNextFromEventId());
// no events
Assert.assertEquals(0, eventInfo.getEvents().size());
int fromEventId = 0;
// source task id will match. all events will be returned
// max events is respected.
when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
for (int i = 0; i < 11; ++i) {
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals((i + 1), fromEventId);
Assert.assertEquals(1, eventInfo.getEvents().size());
}
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
// all events traversed
Assert.assertEquals(11, eventInfo.getNextFromEventId());
// no events
Assert.assertEquals(0, eventInfo.getEvents().size());
// ask for events with sufficient buffer. get all events in a single shot.
fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals(11, fromEventId);
Assert.assertEquals(11, eventInfo.getEvents().size());
// change max events to larger value. max events does not evenly divide total events
fromEventId = 0;
for (int i = 1; i <= 2; ++i) {
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals((i * 5), fromEventId);
Assert.assertEquals(5, eventInfo.getEvents().size());
}
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
// all events traversed
Assert.assertEquals(11, eventInfo.getNextFromEventId());
// remainder events
Assert.assertEquals(1, eventInfo.getEvents().size());
// return more events that dont evenly fit in max size
mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
fromEventId = 0;
int lastFromEventId = 0;
for (int i = 1; i <= 4; ++i) {
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals((i % 2 > 0 ? (lastFromEventId += 2) : (lastFromEventId += 3)), fromEventId);
Assert.assertEquals(5, eventInfo.getEvents().size());
}
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
// all events traversed
Assert.assertEquals(11, eventInfo.getNextFromEventId());
// remainder events
Assert.assertEquals(2, eventInfo.getEvents().size());
}
use of org.apache.tez.dag.app.TaskAttemptEventInfo in project tez by apache.
the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.
@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
initAllVertices(VertexState.INITED);
VertexImpl v1 = vertices.get("vertex1");
startVertex(v1);
VertexImpl v2 = vertices.get("vertex2");
startVertex(v2);
VertexImpl v3 = vertices.get("vertex3");
VertexImpl v4 = vertices.get("vertex4");
List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
// scheduling start to trigger edge routing to begin
for (int i = 0; i < v4.getTotalTasks(); ++i) {
taskList.add(ScheduleTaskRequest.create(i, null));
}
v4.scheduleTasks(taskList);
Assert.assertEquals(VertexState.RUNNING, v4.getState());
Assert.assertEquals(1, v4.sourceVertices.size());
Edge e = v4.sourceVertices.get(v3);
TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
for (int i = 0; i < 11; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// verify all events have been are in taskEvents
Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
TaskAttemptEventInfo eventInfo;
EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
e.edgeManager = mockPlugin;
when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
// send an input failed event
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
// ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
int fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
fromEventId = eventInfo.getNextFromEventId();
Assert.assertEquals(12, fromEventId);
Assert.assertEquals(1, eventInfo.getEvents().size());
Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
// Let failed task send more event
for (int i = 11; i < 14; ++i) {
v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
}
dispatcher.await();
// 11 events + 1 INPUT_FAILED_EVENT.
// Events sent out later by failed tasks should not be available.
Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
fromEventId = 0;
eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}
Aggregations