Examples with DataMovementEvent org.apache.tez.runtime.api.events.DataMovementEvent used on opensource projects

Example 11 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteDataMovementEventToDestination.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteDataMovementEventToDestination() {
    setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0]));
    TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name()));
}

Example 12 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestEdge method testOneToOneEdgeManager.

@Test(timeout = 5000)
public void testOneToOneEdgeManager() {
    EdgeManagerPluginContext mockContext = mock(EdgeManagerPluginContext.class);
    when(mockContext.getSourceVertexName()).thenReturn("Source");
    when(mockContext.getDestinationVertexName()).thenReturn("Destination");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(3);
    OneToOneEdgeManager manager = new OneToOneEdgeManager(mockContext);
    manager.initialize();
    Map<Integer, List<Integer>> destinationTaskAndInputIndices = Maps.newHashMap();
    DataMovementEvent event = DataMovementEvent.create(1, null);
    // fail when source and destination are inconsistent
    when(mockContext.getDestinationVertexNumTasks()).thenReturn(4);
    try {
        manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices);
        Assert.fail();
    } catch (IllegalStateException e) {
        Assert.assertTrue(e.getMessage().contains("1-1 source and destination task counts must match"));
    }
    // now make it consistent
    when(mockContext.getDestinationVertexNumTasks()).thenReturn(3);
    manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices);
    Assert.assertEquals(1, destinationTaskAndInputIndices.size());
    Assert.assertEquals(1, destinationTaskAndInputIndices.entrySet().iterator().next().getKey().intValue());
    Assert.assertEquals(0, destinationTaskAndInputIndices.entrySet().iterator().next().getValue().get(0).intValue());
}

Example 13 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestEdge method testCompositeEventHandling.

@SuppressWarnings({ "rawtypes" })
@Test(timeout = 5000)
public void testCompositeEventHandling() throws TezException {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration());
    TezVertexID srcVertexID = createVertexID(1);
    TezVertexID destVertexID = createVertexID(2);
    LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
    LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
    TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
    Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
    Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
    edge.setSourceVertex(srcVertex);
    edge.setDestinationVertex(destVertex);
    edge.initialize();
    // Task0, Attempt 0
    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2);
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes()));
    // AttemptNum
    cdmEvent.setVersion(2);
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.
    edge.sendTezEventToDestinationTasks(tezEvent);
    verifyEvents(srcTAID, destTasks);
    // Same Verification via regular DataMovementEvents
    // Reset the mock
    resetTaskMocks(destTasks.values());
    for (int i = 0; i < destTasks.size(); i++) {
        DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
        dmEvent.setVersion(2);
        tezEvent = new TezEvent(dmEvent, srcMeta);
        edge.sendTezEventToDestinationTasks(tezEvent);
    }
    verifyEvents(srcTAID, destTasks);
}

Example 14 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class ShuffleInputEventHandlerOrderedGrouped method handleEvent.

private void handleEvent(Event event) throws IOException {
    if (event instanceof DataMovementEvent) {
        numDmeEvents.incrementAndGet();
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet);
        scheduler.updateEventReceivedTime();
    } else if (event instanceof CompositeRoutedDataMovementEvent) {
        CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        if (compositeFetch) {
            numDmeEvents.addAndGet(crdme.getCount());
            processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet);
        } else {
            for (int offset = 0; offset < crdme.getCount(); offset++) {
                numDmeEvents.incrementAndGet();
                processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
            }
        }
        scheduler.updateEventReceivedTime();
    } else if (event instanceof InputFailedEvent) {
        numObsoletionEvents.incrementAndGet();
        processTaskFailedEvent((InputFailedEvent) event);
    }
    if (numDmeEvents.get() + numObsoletionEvents.get() > nextToLogEventCount.get()) {
        logProgress(false);
        // Log every 50 events seen.
        nextToLogEventCount.addAndGet(50);
    }
}

Example 15 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class ShuffleUtils method generateEventsForNonStartedOutput.

/**
 * Generate events for outputs which have not been started.
 * @param eventList
 * @param numPhysicalOutputs
 * @param context
 * @param generateVmEvent whether to generate a vm event or not
 * @param isCompositeEvent whether to generate a CompositeDataMovementEvent or a DataMovementEvent
 * @param deflater
 * @throws IOException
 */
public static void generateEventsForNonStartedOutput(List<Event> eventList, int numPhysicalOutputs, OutputContext context, boolean generateVmEvent, boolean isCompositeEvent, Deflater deflater) throws IOException {
    DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto.newBuilder();
    // Construct the VertexManager event if required.
    if (generateVmEvent) {
        ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder();
        vmBuilder.setOutputSize(0);
        VertexManagerEvent vmEvent = VertexManagerEvent.create(context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
        eventList.add(vmEvent);
    }
    // Construct the DataMovementEvent
    // Always set empty partition information since no files were generated.
    LOG.info("Setting all {} partitions as empty for non-started output", numPhysicalOutputs);
    BitSet emptyPartitionDetails = new BitSet(numPhysicalOutputs);
    emptyPartitionDetails.set(0, numPhysicalOutputs, true);
    ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtilsInternal.toByteArray(emptyPartitionDetails), deflater);
    payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
    payloadBuilder.setRunDuration(0);
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
    ByteBuffer dmePayload = payloadProto.toByteString().asReadOnlyByteBuffer();
    if (isCompositeEvent) {
        CompositeDataMovementEvent cdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, dmePayload);
        eventList.add(cdme);
    } else {
        DataMovementEvent dme = DataMovementEvent.create(0, dmePayload);
        eventList.add(dme);
    }
}