Examples with TezEvent org.apache.tez.runtime.api.impl.TezEvent used on opensource projects

Example 71 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class LogicalIOProcessorRuntimeTask method startRouterThread.

private void startRouterThread() {
    eventRouterThread = new Thread(new RunnableWithNdc() {

        public void runInternal() {
            while (!isTaskDone() && !Thread.currentThread().isInterrupted()) {
                try {
                    TezEvent e = eventsToBeProcessed.take();
                    if (e == null) {
                        continue;
                    }
                    if (!handleEvent(e)) {
                        LOG.warn("Stopping Event Router thread as failed to handle" + " event: " + e);
                        return;
                    }
                } catch (InterruptedException e) {
                    if (!isTaskDone()) {
                        LOG.warn("Event Router thread interrupted. Returning.");
                    }
                    Thread.currentThread().interrupt();
                    return;
                }
            }
        }
    });
    eventRouterThread.setName("TezTaskEventRouter{" + taskSpec.getTaskAttemptID().toString() + "}");
    eventRouterThread.start();
}

Example 72 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class VertexImpl method initializeVertex.

private boolean initializeVertex() {
    // Don't need to initialize committer if vertex is fully completed
    if (recoveryData != null && recoveryData.shouldSkipInit()) {
        // Do other necessary recovery here
        initedTime = recoveryData.getVertexInitedEvent().getInitedTime();
        List<TezEvent> initGeneratedEvents = recoveryData.getVertexInitedEvent().getInitGeneratedEvents();
        if (initGeneratedEvents != null && !initGeneratedEvents.isEmpty()) {
            eventHandler.handle(new VertexEventRouteEvent(getVertexId(), initGeneratedEvents));
        }
        // reset rootInputDescriptor because it may be changed during input initialization.
        this.rootInputDescriptors = recoveryData.getVertexInitedEvent().getAdditionalInputs();
    } else {
        initedTime = clock.getTime();
    }
    // state in recovery mode
    if (recoveryData == null || recoveryData.getVertexFinishedEvent() == null) {
        try {
            initializeCommitters();
        } catch (Exception e) {
            LOG.warn("Vertex Committer init failed, vertex=" + logIdentifier, e);
            addDiagnostic("Vertex init failed : " + ExceptionUtils.getStackTrace(e));
            trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
            finished(VertexState.FAILED);
            return false;
        }
    }
    logJobHistoryVertexInitializedEvent();
    return true;
}

Example 73 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class Edge method stopEventBuffering.

public void stopEventBuffering() throws AMUserCodeException {
    // assume only 1 entity will start and stop event buffering
    bufferEvents.set(false);
    for (TezEvent event : destinationEventBuffer) {
        sendTezEventToDestinationTasks(event);
    }
    destinationEventBuffer.clear();
    for (TezEvent event : sourceEventBuffer) {
        sendTezEventToSourceTasks(event);
    }
    sourceEventBuffer.clear();
}

Example 74 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}

Example 75 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TaskAttemptImpl method sendInputFailedToConsumers.

@VisibleForTesting
protected void sendInputFailedToConsumers() {
    Vertex vertex = getVertex();
    Map<Vertex, Edge> edges = vertex.getOutputVertices();
    if (edges != null && !edges.isEmpty()) {
        List<TezEvent> tezIfEvents = Lists.newArrayListWithCapacity(edges.size());
        for (Vertex edgeVertex : edges.keySet()) {
            tezIfEvents.add(new TezEvent(new InputFailedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, vertex.getName(), edgeVertex.getName(), getID()), appContext.getClock().getTime()));
        }
        sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents));
    }
}