Examples with DataMovementEvent org.apache.tez.runtime.api.events.DataMovementEvent used on opensource projects

Example 1 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class Edge method sendDmEventOrIfEventToTasks.

void sendDmEventOrIfEventToTasks(TezEvent tezEvent, int srcTaskIndex, boolean isDataMovementEvent, Map<Integer, List<Integer>> taskAndInputIndices) {
    Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time");
    Event event = tezEvent.getEvent();
    // cache of event object per input index
    Map<Integer, TezEvent> inputIndicesWithEvents = Maps.newHashMap();
    for (Map.Entry<Integer, List<Integer>> entry : taskAndInputIndices.entrySet()) {
        int destTaskIndex = entry.getKey();
        List<Integer> inputIndices = entry.getValue();
        for (int i = 0; i < inputIndices.size(); ++i) {
            Integer inputIndex = inputIndices.get(i);
            TezEvent tezEventToSend = inputIndicesWithEvents.get(inputIndex);
            if (tezEventToSend == null) {
                Event e;
                if (isDataMovementEvent) {
                    DataMovementEvent dmEvent = (DataMovementEvent) event;
                    e = DataMovementEvent.create(dmEvent.getSourceIndex(), inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload());
                } else {
                    InputFailedEvent ifEvent = ((InputFailedEvent) event);
                    e = InputFailedEvent.create(inputIndex, ifEvent.getVersion());
                }
                tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                // cache the event object per input because are unique per input index
                inputIndicesWithEvents.put(inputIndex, tezEventToSend);
            }
            Task destTask = destinationVertex.getTask(destTaskIndex);
            if (destTask == null) {
                throw new TezUncheckedException("Unexpected null task." + " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcTaskIndex = " + srcTaskIndex + " destVertex=" + destinationVertex.getLogIdentifier() + " destTaskIndex=" + destTaskIndex + " destNumTasks=" + destinationVertex.getTotalTasks() + " edgeManager=" + edgeManager.getClass().getName());
            }
            sendEventToTask(destTask, tezEventToSend);
        }
    }
}

Example 2 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class Edge method handleCompositeDataMovementEvent.

private void handleCompositeDataMovementEvent(TezEvent tezEvent) throws AMUserCodeException {
    CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
    EventMetaData srcInfo = tezEvent.getSourceInfo();
    for (DataMovementEvent dmEvent : compEvent.getEvents()) {
        TezEvent newEvent = new TezEvent(dmEvent, srcInfo, tezEvent.getEventReceivedTime());
        sendTezEventToDestinationTasks(newEvent);
    }
}

Example 3 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestEdge method verifyEvents.

private void verifyEvents(TezTaskAttemptID srcTAID, LinkedHashMap<TezTaskID, Task> destTasks) {
    int count = 0;
    for (Entry<TezTaskID, Task> taskEntry : destTasks.entrySet()) {
        Task mockTask = taskEntry.getValue();
        ArgumentCaptor<TezEvent> args = ArgumentCaptor.forClass(TezEvent.class);
        verify(mockTask, times(1)).registerTezEvent(args.capture());
        TezEvent capturedEvent = args.getValue();
        DataMovementEvent dmEvent = (DataMovementEvent) capturedEvent.getEvent();
        assertEquals(srcTAID.getId(), dmEvent.getVersion());
        assertEquals(count++, dmEvent.getSourceIndex());
        assertEquals(srcTAID.getTaskID().getId(), dmEvent.getTargetIndex());
        byte[] res = new byte[dmEvent.getUserPayload().limit() - dmEvent.getUserPayload().position()];
        dmEvent.getUserPayload().slice().get(res);
        assertTrue(Arrays.equals("bytes".getBytes(), res));
    }
}

Example 4 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestEdge method testOneToOneEdgeManagerODR.

@Test(timeout = 5000)
public void testOneToOneEdgeManagerODR() {
    EdgeManagerPluginContext mockContext = mock(EdgeManagerPluginContext.class);
    when(mockContext.getSourceVertexName()).thenReturn("Source");
    when(mockContext.getDestinationVertexName()).thenReturn("Destination");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(3);
    OneToOneEdgeManagerOnDemand manager = new OneToOneEdgeManagerOnDemand(mockContext);
    manager.initialize();
    Map<Integer, List<Integer>> destinationTaskAndInputIndices = Maps.newHashMap();
    DataMovementEvent event = DataMovementEvent.create(1, null);
    // fail when source and destination are inconsistent
    when(mockContext.getDestinationVertexNumTasks()).thenReturn(4);
    try {
        manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices);
        Assert.fail();
    } catch (IllegalStateException e) {
        Assert.assertTrue(e.getMessage().contains("1-1 source and destination task counts must match"));
    }
    // now make it consistent
    when(mockContext.getDestinationVertexNumTasks()).thenReturn(3);
    manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices);
    Assert.assertEquals(1, destinationTaskAndInputIndices.size());
    Assert.assertEquals(1, destinationTaskAndInputIndices.entrySet().iterator().next().getKey().intValue());
    Assert.assertEquals(0, destinationTaskAndInputIndices.entrySet().iterator().next().getValue().get(0).intValue());
}

Example 5 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestTaskImpl method sendTezEventsToTask.

private void sendTezEventsToTask(TezTaskID taskId, int numTezEvents) {
    EventMetaData eventMetaData = new EventMetaData();
    DataMovementEvent dmEvent = DataMovementEvent.create(null);
    TezEvent tezEvent = new TezEvent(dmEvent, eventMetaData);
    for (int i = 0; i < numTezEvents; i++) {
        mockTask.registerTezEvent(tezEvent);
    }
}