Search in sources :

Example 1 with VertexEventManagerUserCodeError

use of org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError in project tez by apache.

the class VertexImpl method getTaskAttemptTezEvents.

@Override
public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, int fromEventId, int preRoutedFromEventId, int maxEvents) {
    Task task = getTask(attemptID.getTaskID());
    ArrayList<TezEvent> events = task.getTaskAttemptTezEvents(attemptID, preRoutedFromEventId, maxEvents);
    int nextPreRoutedFromEventId = preRoutedFromEventId + events.size();
    int nextFromEventId = fromEventId;
    onDemandRouteEventsReadLock.lock();
    try {
        int currEventCount = onDemandRouteEvents.size();
        try {
            if (currEventCount > fromEventId) {
                if (events != TaskImpl.EMPTY_TASK_ATTEMPT_TEZ_EVENTS) {
                    events.ensureCapacity(maxEvents);
                } else {
                    events = Lists.newArrayListWithCapacity(maxEvents);
                }
                int numPreRoutedEvents = events.size();
                int taskIndex = attemptID.getTaskID().getId();
                Preconditions.checkState(taskIndex < tasks.size(), "Invalid task index for TA: " + attemptID + " vertex: " + getLogIdentifier());
                boolean isFirstEvent = true;
                boolean firstEventObsoleted = false;
                for (nextFromEventId = fromEventId; nextFromEventId < currEventCount; ++nextFromEventId) {
                    boolean earlyExit = false;
                    if (events.size() == maxEvents) {
                        break;
                    }
                    EventInfo eventInfo = onDemandRouteEvents.get(nextFromEventId);
                    if (eventInfo.isObsolete) {
                        // ignore obsolete events
                        firstEventObsoleted = true;
                        continue;
                    }
                    TezEvent tezEvent = eventInfo.tezEvent;
                    switch(tezEvent.getEventType()) {
                        case INPUT_FAILED_EVENT:
                        case DATA_MOVEMENT_EVENT:
                        case COMPOSITE_DATA_MOVEMENT_EVENT:
                            {
                                int srcTaskIndex = eventInfo.eventTaskIndex;
                                Edge srcEdge = eventInfo.eventEdge;
                                PendingEventRouteMetadata pendingRoute = null;
                                if (isFirstEvent) {
                                    // the first event is the one that can have pending routes because its expanded
                                    // events had not been completely sent in the last round.
                                    isFirstEvent = false;
                                    pendingRoute = srcEdge.removePendingEvents(attemptID);
                                    if (pendingRoute != null) {
                                        // obsoleted
                                        if (tezEvent != pendingRoute.getTezEvent()) {
                                            Preconditions.checkState(firstEventObsoleted);
                                            // pending routes can be ignored for obsoleted events
                                            pendingRoute = null;
                                        }
                                    }
                                }
                                if (!srcEdge.maybeAddTezEventForDestinationTask(tezEvent, attemptID, srcTaskIndex, events, maxEvents, pendingRoute)) {
                                    // not enough space left for this iteration events.
                                    // Exit and start from here next time
                                    earlyExit = true;
                                }
                            }
                            break;
                        case ROOT_INPUT_DATA_INFORMATION_EVENT:
                            {
                                InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent();
                                if (riEvent.getTargetIndex() == taskIndex) {
                                    events.add(tezEvent);
                                }
                            }
                            break;
                        default:
                            throw new TezUncheckedException("Unexpected event type for task: " + tezEvent.getEventType());
                    }
                    if (earlyExit) {
                        break;
                    }
                }
                int numEventsSent = events.size() - numPreRoutedEvents;
                if (numEventsSent > 0) {
                    StringBuilder builder = new StringBuilder();
                    builder.append("Sending ").append(attemptID).append(" ").append(numEventsSent).append(" events [").append(fromEventId).append(",").append(nextFromEventId).append(") total ").append(currEventCount).append(" ").append(getLogIdentifier());
                    LOG.info(builder.toString());
                }
            }
        } catch (AMUserCodeException e) {
            String msg = "Exception in " + e.getSource() + ", vertex=" + getLogIdentifier();
            LOG.error(msg, e);
            eventHandler.handle(new VertexEventManagerUserCodeError(getVertexId(), e));
            nextFromEventId = fromEventId;
            events.clear();
        }
    } finally {
        onDemandRouteEventsReadLock.unlock();
    }
    if (!events.isEmpty()) {
        for (int i = (events.size() - 1); i >= 0; --i) {
            TezEvent lastEvent = events.get(i);
            // record the last event sent by the AM to the task
            EventType lastEventType = lastEvent.getEventType();
            // if the following changes then critical path logic/recording may need revision
            if (lastEventType == EventType.COMPOSITE_DATA_MOVEMENT_EVENT || lastEventType == EventType.COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT || lastEventType == EventType.DATA_MOVEMENT_EVENT || lastEventType == EventType.ROOT_INPUT_DATA_INFORMATION_EVENT) {
                task.getAttempt(attemptID).setLastEventSent(lastEvent);
                break;
            }
        }
    }
    return new TaskAttemptEventInfo(nextFromEventId, events, nextPreRoutedFromEventId);
}
Also used : TaskEventScheduleTask(org.apache.tez.dag.app.dag.event.TaskEventScheduleTask) Task(org.apache.tez.dag.app.dag.Task) TaskAttemptEventInfo(org.apache.tez.dag.app.TaskAttemptEventInfo) TezUncheckedException(org.apache.tez.dag.api.TezUncheckedException) DAGEventType(org.apache.tez.dag.app.dag.event.DAGEventType) EventType(org.apache.tez.runtime.api.impl.EventType) VertexEventType(org.apache.tez.dag.app.dag.event.VertexEventType) TaskEventType(org.apache.tez.dag.app.dag.event.TaskEventType) TaskLocationHint(org.apache.tez.dag.api.TaskLocationHint) VertexLocationHint(org.apache.tez.dag.api.VertexLocationHint) PendingEventRouteMetadata(org.apache.tez.dag.app.dag.impl.Edge.PendingEventRouteMetadata) VertexEventManagerUserCodeError(org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError) TezEvent(org.apache.tez.runtime.api.impl.TezEvent) TaskAttemptEventInfo(org.apache.tez.dag.app.TaskAttemptEventInfo) InputDataInformationEvent(org.apache.tez.runtime.api.events.InputDataInformationEvent)

Example 2 with VertexEventManagerUserCodeError

use of org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError in project tez by apache.

the class VertexImpl method scheduleTasks.

@Override
public void scheduleTasks(List<ScheduleTaskRequest> tasksToSchedule) {
    try {
        unsetTasksNotYetScheduled();
        // update state under write lock
        writeLock.lock();
        try {
            for (ScheduleTaskRequest task : tasksToSchedule) {
                if (numTasks <= task.getTaskIndex()) {
                    throw new TezUncheckedException("Invalid taskId: " + task.getTaskIndex() + " for vertex: " + logIdentifier);
                }
                TaskLocationHint locationHint = task.getTaskLocationHint();
                if (locationHint != null) {
                    if (taskLocationHints == null) {
                        taskLocationHints = new TaskLocationHint[numTasks];
                    }
                    taskLocationHints[task.getTaskIndex()] = locationHint;
                }
            }
        } finally {
            writeLock.unlock();
        }
        /**
         * read lock is not needed here. For e.g after starting task
         * scheduling on the vertex, it would not change numTasks. Rest of
         * the methods creating remote task specs have their
         * own locking mechanisms. Ref: TEZ-3297
         */
        for (ScheduleTaskRequest task : tasksToSchedule) {
            TezTaskID taskId = TezTaskID.getInstance(vertexId, task.getTaskIndex());
            TaskSpec baseTaskSpec = createRemoteTaskSpec(taskId.getId());
            boolean fromRecovery = recoveryData == null ? false : recoveryData.getTaskRecoveryData(taskId) != null;
            eventHandler.handle(new TaskEventScheduleTask(taskId, baseTaskSpec, getTaskLocationHint(taskId), fromRecovery));
        }
    } catch (AMUserCodeException e) {
        String msg = "Exception in " + e.getSource() + ", vertex=" + getLogIdentifier();
        LOG.error(msg, e);
        // send event to fail the vertex
        eventHandler.handle(new VertexEventManagerUserCodeError(getVertexId(), e));
        // throw an unchecked exception to stop the vertex manager that invoked this.
        throw new TezUncheckedException(e);
    }
}
Also used : TaskLocationHint(org.apache.tez.dag.api.TaskLocationHint) TezUncheckedException(org.apache.tez.dag.api.TezUncheckedException) VertexEventManagerUserCodeError(org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError) TaskSpec(org.apache.tez.runtime.api.impl.TaskSpec) TaskEventScheduleTask(org.apache.tez.dag.app.dag.event.TaskEventScheduleTask) ScheduleTaskRequest(org.apache.tez.dag.api.VertexManagerPluginContext.ScheduleTaskRequest) TezTaskID(org.apache.tez.dag.records.TezTaskID)

Aggregations

TaskLocationHint (org.apache.tez.dag.api.TaskLocationHint)2 TezUncheckedException (org.apache.tez.dag.api.TezUncheckedException)2 TaskEventScheduleTask (org.apache.tez.dag.app.dag.event.TaskEventScheduleTask)2 VertexEventManagerUserCodeError (org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError)2 VertexLocationHint (org.apache.tez.dag.api.VertexLocationHint)1 ScheduleTaskRequest (org.apache.tez.dag.api.VertexManagerPluginContext.ScheduleTaskRequest)1 TaskAttemptEventInfo (org.apache.tez.dag.app.TaskAttemptEventInfo)1 Task (org.apache.tez.dag.app.dag.Task)1 DAGEventType (org.apache.tez.dag.app.dag.event.DAGEventType)1 TaskEventType (org.apache.tez.dag.app.dag.event.TaskEventType)1 VertexEventType (org.apache.tez.dag.app.dag.event.VertexEventType)1 PendingEventRouteMetadata (org.apache.tez.dag.app.dag.impl.Edge.PendingEventRouteMetadata)1 TezTaskID (org.apache.tez.dag.records.TezTaskID)1 InputDataInformationEvent (org.apache.tez.runtime.api.events.InputDataInformationEvent)1 EventType (org.apache.tez.runtime.api.impl.EventType)1 TaskSpec (org.apache.tez.runtime.api.impl.TaskSpec)1 TezEvent (org.apache.tez.runtime.api.impl.TezEvent)1