Examples with DataMovementEvent org.apache.tez.runtime.api.events.DataMovementEvent used on opensource projects

Example 16 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class ShuffleInputEventHandlerImpl method handleEvent.

private void handleEvent(Event event) throws IOException {
    if (event instanceof DataMovementEvent) {
        numDmeEvents.incrementAndGet();
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet);
        shuffleManager.updateEventReceivedTime();
    } else if (event instanceof CompositeRoutedDataMovementEvent) {
        CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        if (compositeFetch) {
            numDmeEvents.addAndGet(crdme.getCount());
            processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet);
        } else {
            for (int offset = 0; offset < crdme.getCount(); offset++) {
                numDmeEvents.incrementAndGet();
                processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
            }
        }
        shuffleManager.updateEventReceivedTime();
    } else if (event instanceof InputFailedEvent) {
        numObsoletionEvents.incrementAndGet();
        processInputFailedEvent((InputFailedEvent) event);
    } else {
        throw new TezUncheckedException("Unexpected event type: " + event.getClass().getName());
    }
    if (numDmeEvents.get() + numObsoletionEvents.get() > nextToLogEventCount.get()) {
        logProgress(false);
        // Log every 50 events seen.
        nextToLogEventCount.addAndGet(50);
    }
}

Example 17 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TezEvent method serializeEvent.

private void serializeEvent(DataOutput out) throws IOException {
    if (event == null) {
        out.writeBoolean(false);
        return;
    }
    out.writeBoolean(true);
    out.writeInt(eventType.ordinal());
    out.writeLong(eventReceivedTime);
    if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
        // TODO NEWTEZ convert to PB
        TaskStatusUpdateEvent sEvt = (TaskStatusUpdateEvent) event;
        sEvt.write(out);
    } else {
        AbstractMessage message;
        switch(eventType) {
            case CUSTOM_PROCESSOR_EVENT:
                message = ProtoConverters.convertCustomProcessorEventToProto((CustomProcessorEvent) event);
                break;
            case DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertDataMovementEventToProto((DataMovementEvent) event);
                break;
            case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeRoutedDataMovementEventToProto((CompositeRoutedDataMovementEvent) event);
                break;
            case COMPOSITE_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeDataMovementEventToProto((CompositeDataMovementEvent) event);
                break;
            case VERTEX_MANAGER_EVENT:
                message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event);
                break;
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
                message = InputReadErrorEventProto.newBuilder().setIndex(ideEvt.getIndex()).setDiagnostics(ideEvt.getDiagnostics()).setVersion(ideEvt.getVersion()).build();
                break;
            case TASK_ATTEMPT_FAILED_EVENT:
                TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event;
                message = TaskAttemptFailedEventProto.newBuilder().setDiagnostics(tfEvt.getDiagnostics()).setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())).build();
                break;
            case TASK_ATTEMPT_KILLED_EVENT:
                TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event;
                message = TaskAttemptKilledEventProto.newBuilder().setDiagnostics(tkEvent.getDiagnostics()).build();
                break;
            case TASK_ATTEMPT_COMPLETED_EVENT:
                message = TaskAttemptCompletedEventProto.newBuilder().build();
                break;
            case INPUT_FAILED_EVENT:
                InputFailedEvent ifEvt = (InputFailedEvent) event;
                message = InputFailedEventProto.newBuilder().setTargetIndex(ifEvt.getTargetIndex()).setVersion(ifEvt.getVersion()).build();
                break;
            case ROOT_INPUT_DATA_INFORMATION_EVENT:
                message = ProtoConverters.convertRootInputDataInformationEventToProto((InputDataInformationEvent) event);
                break;
            case ROOT_INPUT_INITIALIZER_EVENT:
                message = ProtoConverters.convertRootInputInitializerEventToProto((InputInitializerEvent) event);
                break;
            default:
                throw new TezUncheckedException("Unknown TezEvent" + ", type=" + eventType);
        }
        if (out instanceof OutputStream) {
            // DataOutputBuffer extends DataOutputStream
            int serializedSize = message.getSerializedSize();
            out.writeInt(serializedSize);
            int buffersize = serializedSize < CodedOutputStream.DEFAULT_BUFFER_SIZE ? serializedSize : CodedOutputStream.DEFAULT_BUFFER_SIZE;
            CodedOutputStream codedOut = CodedOutputStream.newInstance((OutputStream) out, buffersize);
            message.writeTo(codedOut);
            codedOut.flush();
        } else {
            byte[] eventBytes = message.toByteArray();
            out.writeInt(eventBytes.length);
            out.write(eventBytes);
        }
    }
}

Example 18 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class Edge method sendTezEventToDestinationTasks.

public void sendTezEventToDestinationTasks(TezEvent tezEvent) throws AMUserCodeException {
    if (!bufferEvents.get()) {
        boolean isDataMovementEvent = true;
        switch(tezEvent.getEventType()) {
            case COMPOSITE_DATA_MOVEMENT_EVENT:
                handleCompositeDataMovementEvent(tezEvent);
                break;
            case INPUT_FAILED_EVENT:
            case DATA_MOVEMENT_EVENT:
                if (tezEvent.getEventType().equals(EventType.INPUT_FAILED_EVENT)) {
                    isDataMovementEvent = false;
                }
                Map<Integer, List<Integer>> destTaskAndInputIndices = Maps.newHashMap();
                TezTaskAttemptID srcAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
                int srcTaskIndex = srcAttemptId.getTaskID().getId();
                boolean routingRequired = routingNeeded;
                if (routingRequired) {
                    try {
                        if (isDataMovementEvent) {
                            DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                            edgeManager.routeDataMovementEventToDestination(dmEvent, srcTaskIndex, dmEvent.getSourceIndex(), destTaskAndInputIndices);
                        } else {
                            edgeManager.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, destTaskAndInputIndices);
                        }
                    } catch (Exception e) {
                        throw new AMUserCodeException(Source.EdgeManager, "Fail to sendTezEventToDestinationTasks, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
                    }
                } else {
                    LOG.info("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
                }
                if (!destTaskAndInputIndices.isEmpty()) {
                    sendDmEventOrIfEventToTasks(tezEvent, srcTaskIndex, isDataMovementEvent, destTaskAndInputIndices);
                } else if (routingRequired) {
                    throw new TezUncheckedException("Event must be routed." + generateCommonDebugString(srcTaskIndex, tezEvent));
                }
                break;
            default:
                throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
        }
    } else {
        destinationEventBuffer.add(tezEvent);
    }
}

Example 19 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}

Example 20 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestInput method handleEvents.

@Override
public void handleEvents(List<Event> inputEvents) throws Exception {
    for (Event event : inputEvents) {
        if (event instanceof DataMovementEvent) {
            DataMovementEvent dmEvent = (DataMovementEvent) event;
            numCompletedInputs++;
            LOG.info(getContext().getSourceVertexName() + " Received DataMovement event sourceId : " + dmEvent.getSourceIndex() + " targetId: " + dmEvent.getTargetIndex() + " version: " + dmEvent.getVersion() + " numInputs: " + getNumPhysicalInputs() + " numCompletedInputs: " + numCompletedInputs);
            this.completedInputVersion[dmEvent.getTargetIndex()] = dmEvent.getVersion();
            this.inputValues[dmEvent.getTargetIndex()] = dmEvent.getUserPayload().getInt();
        } else if (event instanceof InputFailedEvent) {
            InputFailedEvent ifEvent = (InputFailedEvent) event;
            if (this.completedInputVersion[ifEvent.getTargetIndex()] == ifEvent.getVersion()) {
                numCompletedInputs--;
            }
            LOG.info("Received InputFailed event targetId: " + ifEvent.getTargetIndex() + " version: " + ifEvent.getVersion() + " numInputs: " + getNumPhysicalInputs() + " numCompletedInputs: " + numCompletedInputs);
        }
    }
    if (numCompletedInputs == getNumPhysicalInputs()) {
        int maxInputVersionSeen = -1;
        for (int i = 0; i < getNumPhysicalInputs(); ++i) {
            if (completedInputVersion[i] < 0) {
                LOG.info("Not received completion for input " + i);
                return;
            } else if (maxInputVersionSeen < completedInputVersion[i]) {
                maxInputVersionSeen = completedInputVersion[i];
            }
        }
        LOG.info("Received all inputs");
        synchronized (inputReady) {
            inputReady.set(maxInputVersionSeen);
            LOG.info("Notifying done with " + maxInputVersionSeen);
            inputReady.notifyAll();
        }
    }
}