Examples with CompositeRoutedDataMovementEvent org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent used on opensource projects

Example 1 with CompositeRoutedDataMovementEvent

use of org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent in project tez by apache.

the class TestMockDAGAppMaster method testBasicEvents.

@Test(timeout = 5000)
public void testBasicEvents() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testBasicEvents");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    List<TezEvent> tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 1 from vA
    Assert.assertEquals(1, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    tezClient.stop();
}

Example 2 with CompositeRoutedDataMovementEvent

use of org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent in project tez by apache.

the class ShuffleInputEventHandlerOrderedGrouped method handleEvent.

private void handleEvent(Event event) throws IOException {
    if (event instanceof DataMovementEvent) {
        numDmeEvents.incrementAndGet();
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet);
        scheduler.updateEventReceivedTime();
    } else if (event instanceof CompositeRoutedDataMovementEvent) {
        CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        if (compositeFetch) {
            numDmeEvents.addAndGet(crdme.getCount());
            processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet);
        } else {
            for (int offset = 0; offset < crdme.getCount(); offset++) {
                numDmeEvents.incrementAndGet();
                processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
            }
        }
        scheduler.updateEventReceivedTime();
    } else if (event instanceof InputFailedEvent) {
        numObsoletionEvents.incrementAndGet();
        processTaskFailedEvent((InputFailedEvent) event);
    }
    if (numDmeEvents.get() + numObsoletionEvents.get() > nextToLogEventCount.get()) {
        logProgress(false);
        // Log every 50 events seen.
        nextToLogEventCount.addAndGet(50);
    }
}

Example 3 with CompositeRoutedDataMovementEvent

use of org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent in project tez by apache.

the class ShuffleInputEventHandlerImpl method handleEvent.

private void handleEvent(Event event) throws IOException {
    if (event instanceof DataMovementEvent) {
        numDmeEvents.incrementAndGet();
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet);
        shuffleManager.updateEventReceivedTime();
    } else if (event instanceof CompositeRoutedDataMovementEvent) {
        CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        if (compositeFetch) {
            numDmeEvents.addAndGet(crdme.getCount());
            processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet);
        } else {
            for (int offset = 0; offset < crdme.getCount(); offset++) {
                numDmeEvents.incrementAndGet();
                processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
            }
        }
        shuffleManager.updateEventReceivedTime();
    } else if (event instanceof InputFailedEvent) {
        numObsoletionEvents.incrementAndGet();
        processInputFailedEvent((InputFailedEvent) event);
    } else {
        throw new TezUncheckedException("Unexpected event type: " + event.getClass().getName());
    }
    if (numDmeEvents.get() + numObsoletionEvents.get() > nextToLogEventCount.get()) {
        logProgress(false);
        // Log every 50 events seen.
        nextToLogEventCount.addAndGet(50);
    }
}

Example 4 with CompositeRoutedDataMovementEvent

use of org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent in project tez by apache.

the class TezEvent method serializeEvent.

private void serializeEvent(DataOutput out) throws IOException {
    if (event == null) {
        out.writeBoolean(false);
        return;
    }
    out.writeBoolean(true);
    out.writeInt(eventType.ordinal());
    out.writeLong(eventReceivedTime);
    if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
        // TODO NEWTEZ convert to PB
        TaskStatusUpdateEvent sEvt = (TaskStatusUpdateEvent) event;
        sEvt.write(out);
    } else {
        AbstractMessage message;
        switch(eventType) {
            case CUSTOM_PROCESSOR_EVENT:
                message = ProtoConverters.convertCustomProcessorEventToProto((CustomProcessorEvent) event);
                break;
            case DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertDataMovementEventToProto((DataMovementEvent) event);
                break;
            case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeRoutedDataMovementEventToProto((CompositeRoutedDataMovementEvent) event);
                break;
            case COMPOSITE_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeDataMovementEventToProto((CompositeDataMovementEvent) event);
                break;
            case VERTEX_MANAGER_EVENT:
                message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event);
                break;
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
                message = InputReadErrorEventProto.newBuilder().setIndex(ideEvt.getIndex()).setDiagnostics(ideEvt.getDiagnostics()).setVersion(ideEvt.getVersion()).build();
                break;
            case TASK_ATTEMPT_FAILED_EVENT:
                TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event;
                message = TaskAttemptFailedEventProto.newBuilder().setDiagnostics(tfEvt.getDiagnostics()).setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())).build();
                break;
            case TASK_ATTEMPT_KILLED_EVENT:
                TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event;
                message = TaskAttemptKilledEventProto.newBuilder().setDiagnostics(tkEvent.getDiagnostics()).build();
                break;
            case TASK_ATTEMPT_COMPLETED_EVENT:
                message = TaskAttemptCompletedEventProto.newBuilder().build();
                break;
            case INPUT_FAILED_EVENT:
                InputFailedEvent ifEvt = (InputFailedEvent) event;
                message = InputFailedEventProto.newBuilder().setTargetIndex(ifEvt.getTargetIndex()).setVersion(ifEvt.getVersion()).build();
                break;
            case ROOT_INPUT_DATA_INFORMATION_EVENT:
                message = ProtoConverters.convertRootInputDataInformationEventToProto((InputDataInformationEvent) event);
                break;
            case ROOT_INPUT_INITIALIZER_EVENT:
                message = ProtoConverters.convertRootInputInitializerEventToProto((InputInitializerEvent) event);
                break;
            default:
                throw new TezUncheckedException("Unknown TezEvent" + ", type=" + eventType);
        }
        if (out instanceof OutputStream) {
            // DataOutputBuffer extends DataOutputStream
            int serializedSize = message.getSerializedSize();
            out.writeInt(serializedSize);
            int buffersize = serializedSize < CodedOutputStream.DEFAULT_BUFFER_SIZE ? serializedSize : CodedOutputStream.DEFAULT_BUFFER_SIZE;
            CodedOutputStream codedOut = CodedOutputStream.newInstance((OutputStream) out, buffersize);
            message.writeTo(codedOut);
            codedOut.flush();
        } else {
            byte[] eventBytes = message.toByteArray();
            out.writeInt(eventBytes.length);
            out.write(eventBytes);
        }
    }
}

Example 5 with CompositeRoutedDataMovementEvent

use of org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}