Examples with EdgeManagerPluginOnDemand org.apache.tez.dag.api.EdgeManagerPluginOnDemand used on opensource projects

Example 6 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
    // Let failed task send more event
    for (int i = 11; i < 14; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // 11 events + 1 INPUT_FAILED_EVENT.
    // Events sent out later by failed tasks should not be available.
    Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 7 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}

Example 8 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestFairShuffleVertexManager method testFairSchedulingWithPartitionStats.

@Test(timeout = 5000)
public void testFairSchedulingWithPartitionStats() throws Exception {
    final int numScatherAndGatherSourceTasks = 300;
    final Map<String, EdgeManagerPlugin> newEdgeManagers = new HashMap<String, EdgeManagerPlugin>();
    long[] partitionStats = new long[] { (MB), (2 * MB), (5 * MB) };
    testSchedulingWithPartitionStats(FairRoutingType.FAIR_PARALLELISM, numScatherAndGatherSourceTasks, partitionStats, 2, 3, 2, newEdgeManagers);
    // Get the first edgeManager which is SCATTER_GATHER.
    EdgeManagerPluginOnDemand edgeManager = (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next();
    // The first destination task fetches two partitions from all source tasks.
    // Thus the # of inputs == # of source tasks * 2 merged partitions
    Assert.assertEquals(numScatherAndGatherSourceTasks * 2, edgeManager.getNumDestinationTaskPhysicalInputs(0));
    for (int sourceTaskIndex = 0; sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) {
        for (int j = 0; j < 2; j++) {
            if (j == 0) {
                EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(sourceTaskIndex, 0);
                Assert.assertEquals(2, routeMetadata.getCount());
                Assert.assertEquals(0, routeMetadata.getSource());
                Assert.assertEquals(sourceTaskIndex * 2, routeMetadata.getTarget());
            } else {
                EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(sourceTaskIndex, 0);
                Assert.assertEquals(2, routeMetadata.getNumEvents());
                Assert.assertArrayEquals(new int[] { 0 + sourceTaskIndex * 2, 1 + sourceTaskIndex * 2 }, routeMetadata.getTargetIndices());
            }
        }
    }
    // The 2nd destination task fetches one partition from the first half of
    // source tasks.
    Assert.assertEquals(numScatherAndGatherSourceTasks / 2, edgeManager.getNumDestinationTaskPhysicalInputs(1));
    for (int j = 0; j < 2; j++) {
        if (j == 0) {
            EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(0, 1);
            Assert.assertEquals(1, routeMetadata.getCount());
            Assert.assertEquals(2, routeMetadata.getSource());
            Assert.assertEquals(0, routeMetadata.getTarget());
        } else {
            EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(0, 1);
            Assert.assertEquals(1, routeMetadata.getNumEvents());
            Assert.assertEquals(0, routeMetadata.getTargetIndices()[0]);
        }
    }
    // The 3rd destination task fetches one partition from 2nd half of
    // source tasks.
    Assert.assertEquals(numScatherAndGatherSourceTasks / 2, edgeManager.getNumDestinationTaskPhysicalInputs(2));
    for (int sourceTaskIndex = numScatherAndGatherSourceTasks / 2; sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) {
        for (int j = 0; j < 2; j++) {
            if (j == 0) {
                EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(sourceTaskIndex, 2);
                Assert.assertEquals(1, routeMetadata.getCount());
                Assert.assertEquals(2, routeMetadata.getSource());
                Assert.assertEquals(sourceTaskIndex - numScatherAndGatherSourceTasks / 2, routeMetadata.getTarget());
            } else {
                EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(sourceTaskIndex, 2);
                Assert.assertEquals(1, routeMetadata.getNumEvents());
                Assert.assertEquals(sourceTaskIndex - numScatherAndGatherSourceTasks / 2, routeMetadata.getTargetIndices()[0]);
            }
        }
    }
}

Example 9 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestFairShuffleVertexManager method testSchedulingWithPartitionStats.

// Create a DAG with one destination vertexes connected to 3 source vertexes.
// There are 3 tasks for each vertex. One edge is of type SCATTER_GATHER.
// The other edges are BROADCAST.
private void testSchedulingWithPartitionStats(FairRoutingType fairRoutingType, int numTasks, long[] partitionStats, int numCompletedEvents, int expectedScheduledTasks, int expectedNumDestinationConsumerTasks, Map<String, EdgeManagerPlugin> newEdgeManagers) throws Exception {
    Configuration conf = new Configuration();
    FairShuffleVertexManager manager;
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String r1 = "R1";
    final int numOfTasksInr1 = numTasks;
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String m2 = "M2";
    final int numOfTasksInM2 = 3;
    EdgeProperty eProp2 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String m3 = "M3";
    final int numOfTasksInM3 = 3;
    EdgeProperty eProp3 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    final String mockManagedVertexId = "R2";
    final int numOfTasksInDestination = 3;
    mockInputVertices.put(r1, eProp1);
    mockInputVertices.put(m2, eProp2);
    mockInputVertices.put(m3, eProp3);
    final VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(numOfTasksInDestination);
    when(mockContext.getVertexNumTasks(r1)).thenReturn(numOfTasksInr1);
    when(mockContext.getVertexNumTasks(m2)).thenReturn(numOfTasksInM2);
    when(mockContext.getVertexNumTasks(m3)).thenReturn(numOfTasksInM3);
    final List<Integer> scheduledTasks = Lists.newLinkedList();
    doAnswer(new ScheduledTasksAnswer(scheduledTasks)).when(mockContext).scheduleTasks(anyList());
    doAnswer(new reconfigVertexAnswer(mockContext, mockManagedVertexId, newEdgeManagers)).when(mockContext).reconfigureVertex(anyInt(), any(VertexLocationHint.class), anyMap());
    // check initialization
    manager = createFairShuffleVertexManager(conf, mockContext, fairRoutingType, 1000 * MB, 0.001f, 0.001f);
    manager.onVertexStarted(emptyCompletions);
    Assert.assertTrue(manager.bipartiteSources == 1);
    manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(m2, VertexState.CONFIGURED));
    Assert.assertEquals(numOfTasksInDestination, // no tasks scheduled
    manager.pendingTasks.size());
    Assert.assertEquals(numOfTasksInr1, manager.totalNumBipartiteSourceTasks);
    Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
    // no tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == numOfTasksInDestination);
    Assert.assertTrue(manager.totalNumBipartiteSourceTasks == numOfTasksInr1);
    for (int i = 0; i < numCompletedEvents; i++) {
        VertexManagerEvent vmEvent = getVertexManagerEvent(partitionStats, 0, r1, true);
        manager.onSourceTaskCompleted(vmEvent.getProducerAttemptIdentifier());
        // send VM event
        manager.onVertexManagerEventReceived(vmEvent);
    }
    // Send an event for m2.
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(m2, 0));
    // no tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == numOfTasksInDestination);
    Assert.assertTrue(manager.totalNumBipartiteSourceTasks == numOfTasksInr1);
    // Send an event for m3.
    manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(m3, 0));
    // all tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == 0);
    Assert.assertTrue(scheduledTasks.size() == expectedScheduledTasks);
    Assert.assertEquals(1, newEdgeManagers.size());
    EdgeManagerPluginOnDemand edgeManager = (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next();
    // the same as original number of partitions.
    for (int i = 0; i < numOfTasksInr1; i++) {
        Assert.assertEquals(numOfTasksInDestination, edgeManager.getNumSourceTaskPhysicalOutputs(0));
    }
    for (int sourceTaskIndex = 0; sourceTaskIndex < numOfTasksInr1; sourceTaskIndex++) {
        Assert.assertEquals(expectedNumDestinationConsumerTasks, edgeManager.getNumDestinationConsumerTasks(sourceTaskIndex));
    }
}