Examples with EdgeManagerPluginOnDemand org.apache.tez.dag.api.EdgeManagerPluginOnDemand used on opensource projects

Example 1 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class Edge method routingToBegin.

public void routingToBegin() throws AMUserCodeException {
    int numDestTasks = edgeManagerContext.getDestinationVertexNumTasks();
    synchronized (this) {
        if (numDestTasks == 0) {
            routingNeeded = false;
        } else if (numDestTasks < 0) {
            throw new TezUncheckedException("Internal error. Not expected to route events to a destination until parallelism is determined" + " sourceVertex=" + sourceVertex.getLogIdentifier() + " edgeManager=" + edgeManager.getClass().getName());
        }
        if (edgeManager instanceof EdgeManagerPluginOnDemand) {
            onDemandRouting = true;
        }
    }
    if (onDemandRouting) {
        try {
            ((EdgeManagerPluginOnDemand) edgeManager).prepareForRouting();
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to prepareForRouting " + getEdgeInfo(), e);
        }
    }
    LOG.info("Routing to begin for edge: " + getEdgeInfo() + ". EdgeProperty: " + edgeProperty + " onDemandRouting: " + hasOnDemandRouting());
}

Example 2 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestFairShuffleVertexManager method testReduceSchedulingWithPartitionStats.

@Test(timeout = 5000)
public void testReduceSchedulingWithPartitionStats() throws Exception {
    final int numScatherAndGatherSourceTasks = 300;
    final Map<String, EdgeManagerPlugin> newEdgeManagers = new HashMap<String, EdgeManagerPlugin>();
    long[] partitionStats = new long[] { (MB), (2 * MB), (5 * MB) };
    testSchedulingWithPartitionStats(FairRoutingType.REDUCE_PARALLELISM, numScatherAndGatherSourceTasks, partitionStats, 2, 2, 2, newEdgeManagers);
    EdgeManagerPluginOnDemand edgeManager = (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next();
    // The first destination task fetches two partitions from all source tasks.
    // Thus the # of inputs == # of source tasks * 2 merged partitions
    Assert.assertEquals(numScatherAndGatherSourceTasks * 2, edgeManager.getNumDestinationTaskPhysicalInputs(0));
    for (int sourceTaskIndex = 0; sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) {
        for (int j = 0; j < 2; j++) {
            if (j == 0) {
                EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(sourceTaskIndex, 0);
                Assert.assertEquals(2, routeMetadata.getCount());
                Assert.assertEquals(0, routeMetadata.getSource());
                Assert.assertEquals(sourceTaskIndex * 2, routeMetadata.getTarget());
            } else {
                EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(sourceTaskIndex, 0);
                Assert.assertEquals(2, routeMetadata.getNumEvents());
                Assert.assertArrayEquals(new int[] { 0 + sourceTaskIndex * 2, 1 + sourceTaskIndex * 2 }, routeMetadata.getTargetIndices());
            }
        }
    }
}

Example 3 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestShuffleVertexManagerBase method testAutoParallelism.

@Test(timeout = 5000)
public void testAutoParallelism() throws Exception {
    Configuration conf = new Configuration();
    ShuffleVertexManagerBase manager;
    final String mockSrcVertexId1 = "Vertex1";
    final String mockSrcVertexId2 = "Vertex2";
    final String mockSrcVertexId3 = "Vertex3";
    final String mockManagedVertexId = "Vertex4";
    final List<Integer> scheduledTasks = Lists.newLinkedList();
    final Map<String, EdgeManagerPlugin> newEdgeManagers = new HashMap<String, EdgeManagerPlugin>();
    final VertexManagerPluginContext mockContext = createVertexManagerContext(mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, mockManagedVertexId, 4, scheduledTasks, newEdgeManagers);
    // parallelism changed due to small data size
    manager = createManager(conf, mockContext, 0.5f, 0.5f);
    manager.onVertexStarted(emptyCompletions);
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
    // no tasks scheduled
    Assert.assertEquals(4, manager.pendingTasks.size());
    Assert.assertEquals(4, manager.totalNumBipartiteSourceTasks);
    // task completion from non-bipartite stage does nothing
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId3, 0));
    // no tasks scheduled
    Assert.assertEquals(4, manager.pendingTasks.size());
    Assert.assertEquals(4, manager.totalNumBipartiteSourceTasks);
    Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
    VertexManagerEvent vmEvent = getVertexManagerEvent(null, 50 * MB, mockSrcVertexId1);
    manager.onVertexManagerEventReceived(vmEvent);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(4, manager.pendingTasks.size());
    // no tasks scheduled
    Assert.assertEquals(0, scheduledTasks.size());
    Assert.assertEquals(1, manager.numBipartiteSourceTasksCompleted);
    Assert.assertEquals(1, manager.numVertexManagerEventsReceived);
    Assert.assertEquals(50 * MB, manager.completedSourceTasksOutputSize);
    // ignore duplicate completion
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    Assert.assertEquals(4, manager.pendingTasks.size());
    // no tasks scheduled
    Assert.assertEquals(0, scheduledTasks.size());
    Assert.assertEquals(1, manager.numBipartiteSourceTasksCompleted);
    Assert.assertEquals(50 * MB, manager.completedSourceTasksOutputSize);
    vmEvent = getVertexManagerEvent(null, 50 * MB, mockSrcVertexId2);
    manager.onVertexManagerEventReceived(vmEvent);
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    // managedVertex tasks reduced
    verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(VertexLocationHint.class), anyMap());
    verify(mockContext, times(1)).reconfigureVertex(eq(2), any(VertexLocationHint.class), anyMap());
    Assert.assertEquals(2, newEdgeManagers.size());
    // TODO improve tests for parallelism
    // all tasks scheduled
    Assert.assertEquals(0, manager.pendingTasks.size());
    Assert.assertEquals(2, scheduledTasks.size());
    Assert.assertTrue(scheduledTasks.contains(new Integer(0)));
    Assert.assertTrue(scheduledTasks.contains(new Integer(1)));
    Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted);
    Assert.assertEquals(2, manager.numVertexManagerEventsReceived);
    Assert.assertEquals(100 * MB, manager.completedSourceTasksOutputSize);
    // more completions dont cause recalculation of parallelism
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(VertexLocationHint.class), anyMap());
    Assert.assertEquals(2, newEdgeManagers.size());
    EdgeManagerPluginOnDemand edgeManager = (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next();
    // 4 source task outputs - same as original number of partitions
    Assert.assertEquals(4, edgeManager.getNumSourceTaskPhysicalOutputs(0));
    // 4 destination task inputs - 2 source tasks * 2 merged partitions
    Assert.assertEquals(4, edgeManager.getNumDestinationTaskPhysicalInputs(0));
    EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeDataMovementEventToDestination(1, 1, 0);
    Assert.assertEquals(1, routeMetadata.getNumEvents());
    Assert.assertEquals(3, routeMetadata.getTargetIndices()[0]);
    routeMetadata = edgeManager.routeDataMovementEventToDestination(0, 2, 1);
    Assert.assertEquals(1, routeMetadata.getNumEvents());
    Assert.assertEquals(0, routeMetadata.getTargetIndices()[0]);
    routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 0);
    Assert.assertEquals(2, routeMetadata.getNumEvents());
    Assert.assertEquals(2, routeMetadata.getTargetIndices()[0]);
    Assert.assertEquals(3, routeMetadata.getTargetIndices()[1]);
    routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1);
    Assert.assertEquals(2, routeMetadata.getNumEvents());
    Assert.assertEquals(2, routeMetadata.getTargetIndices()[0]);
    Assert.assertEquals(3, routeMetadata.getTargetIndices()[1]);
}

Example 4 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletionWithPendingRoutes.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletionWithPendingRoutes() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockFailedRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockFailedRoute);
    // return more events that dont evenly fit in max size
    EventRouteMetadata mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    fromEventId = eventInfo.getNextFromEventId();
    // 0-1 events expanded and fit, 2nd event has pending routes
    Assert.assertEquals(2, fromEventId);
    Assert.assertEquals(5, eventInfo.getEvents().size());
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // get only input failed event. all DM events obsoleted
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 5 with EdgeManagerPluginOnDemand

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttempts.

@Test(timeout = 5000)
public void testVertexGetTAAttempts() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 5; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been put in pending.
    // this is not necessary after legacy routing has been removed
    Assert.assertEquals(5, v4.pendingTaskEvents.size());
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    dispatcher.await();
    // verify all events have been moved to taskEvents
    Assert.assertEquals(5, v4.getOnDemandRouteEvents().size());
    for (int i = 5; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    // source task id will not match. all events will return null
    when(mockPlugin.routeDataMovementEventToDestination(1, 0, 0)).thenReturn(mockRoute);
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, 0, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    int fromEventId = 0;
    // source task id will match. all events will be returned
    // max events is respected.
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    for (int i = 0; i < 11; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i + 1), fromEventId);
        Assert.assertEquals(1, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // no events
    Assert.assertEquals(0, eventInfo.getEvents().size());
    // ask for events with sufficient buffer. get all events in a single shot.
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(11, fromEventId);
    Assert.assertEquals(11, eventInfo.getEvents().size());
    // change max events to larger value. max events does not evenly divide total events
    fromEventId = 0;
    for (int i = 1; i <= 2; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i * 5), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(1, eventInfo.getEvents().size());
    // return more events that dont evenly fit in max size
    mockRoute = EventRouteMetadata.create(2, new int[] { 0, 0 });
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    fromEventId = 0;
    int lastFromEventId = 0;
    for (int i = 1; i <= 4; ++i) {
        eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
        fromEventId = eventInfo.getNextFromEventId();
        Assert.assertEquals((i % 2 > 0 ? (lastFromEventId += 2) : (lastFromEventId += 3)), fromEventId);
        Assert.assertEquals(5, eventInfo.getEvents().size());
    }
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5);
    // all events traversed
    Assert.assertEquals(11, eventInfo.getNextFromEventId());
    // remainder events
    Assert.assertEquals(2, eventInfo.getEvents().size());
}