Examples with EventRouteMetadata org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata used on opensource projects

Example 6 with EventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletionWithPendingRoutes.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletionWithPendingRoutes() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockFailedRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockFailedRoute);
    // return more events that dont evenly fit in max size
    EventRouteMetadata mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    fromEventId = eventInfo.getNextFromEventId();
    // 0-1 events expanded and fit, 2nd event has pending routes
    Assert.assertEquals(2, fromEventId);
    Assert.assertEquals(5, eventInfo.getEvents().size());
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // get only input failed event. all DM events obsoleted
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 7 with EventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttempts.

@Test(timeout = 5000)
public void testVertexGetTAAttempts() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 5; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been put in pending.
    // this is not necessary after legacy routing has been removed
    Assert.assertEquals(5, v4.pendingTaskEvents.size());
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    dispatcher.await();
    // verify all events have been moved to taskEvents
    Assert.assertEquals(5, v4.getOnDemandRouteEvents().size());
    for (int i = 5; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    // source task id will not match. all events will return null
    when(mockPlugin.routeDataMovementEventToDestination(1, 0, 0)).thenReturn(mockRoute);
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, 0, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    int fromEventId = 0;
    // source task id will match. all events will be returned
    // max events is respected.
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    for (int i = 0; i < 11; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i + 1), fromEventId);
        Assert.assertEquals(1, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    // ask for events with sufficient buffer. get all events in a single shot.
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(11, fromEventId);
    Assert.assertEquals(11, eventInfo.getEvents().size());
    // change max events to larger value. max events does not evenly divide total events
    fromEventId = 0;
    for (int i = 1; i <= 2; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i * 5), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(1, eventInfo.getEvents().size());
    // return more events that dont evenly fit in max size
    mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    fromEventId = 0;
    int lastFromEventId = 0;
    for (int i = 1; i <= 4; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i % 2 > 0 ? (lastFromEventId += 2) : (lastFromEventId += 3)), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(2, eventInfo.getEvents().size());
}

Example 8 with EventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
    // Let failed task send more event
    for (int i = 11; i < 14; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // 11 events + 1 INPUT_FAILED_EVENT.
    // Events sent out later by failed tasks should not be available.
    Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 9 with EventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}

Example 10 with EventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata in project tez by apache.

the class TestCartesianProductEdgeManagerPartitioned method testThreeWayV1.

private void testThreeWayV1(CartesianProductConfigProto config) throws Exception {
    when(mockContext.getSourceVertexName()).thenReturn("v1");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(3);
    edgeManager.initialize(config);
    CompositeEventRouteMetadata compositeRoutingData = edgeManager.routeCompositeDataMovementEventToDestination(1, 1);
    assertNotNull(compositeRoutingData);
    assertEquals(1, compositeRoutingData.getCount());
    assertEquals(0, compositeRoutingData.getSource());
    assertEquals(1, compositeRoutingData.getTarget());
    EventRouteMetadata routingData = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1);
    assertNotNull(routingData);
    assertEquals(1, routingData.getNumEvents());
    assertArrayEquals(new int[] { 1 }, routingData.getTargetIndices());
    assertEquals(2, edgeManager.routeInputErrorEventToSource(1, 2));
    assertEquals(24, edgeManager.getNumDestinationConsumerTasks(1));
    assertEquals(3, edgeManager.getNumDestinationTaskPhysicalInputs(10));
    assertEquals(3, edgeManager.getNumSourceTaskPhysicalOutputs(2));
}