Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 46 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class Edge method setCustomEdgeManager.

// Test only method for creating specific scenarios
@VisibleForTesting
void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) throws AMUserCodeException {
    EdgeProperty modifiedEdgeProperty = EdgeProperty.create(descriptor, edgeProperty.getDataSourceType(), edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(), edgeProperty.getEdgeDestination());
    setEdgeProperty(modifiedEdgeProperty);
}

Example 47 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class RootInputVertexManager method onVertexStarted.

@Override
public void onVertexStarted(List<TaskAttemptIdentifier> completions) {
    Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
    for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) {
        String srcVertex = entry.getKey();
        // track vertices with task count > 0
        int numTasks = getContext().getVertexNumTasks(srcVertex);
        if (numTasks > 0) {
            LOG.info("Task count in " + srcVertex + ": " + numTasks);
            srcVertexInfo.put(srcVertex, createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName())));
            getContext().registerForVertexStateUpdates(srcVertex, EnumSet.of(VertexState.CONFIGURED));
        } else {
            LOG.info("Vertex: " + getContext().getVertexName() + "; Ignoring " + srcVertex + " as it has " + numTasks + " tasks");
        }
    }
    if (completions != null) {
        for (TaskAttemptIdentifier attempt : completions) {
            onSourceTaskCompleted(attempt);
        }
    }
    onVertexStartedDone.set(true);
    // track the tasks in this vertex
    updatePendingTasks();
    processPendingTasks();
}

Example 48 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestFairShuffleVertexManager method testSchedulingWithPartitionStats.

// Create a DAG with one destination vertexes connected to 3 source vertexes.
// There are 3 tasks for each vertex. One edge is of type SCATTER_GATHER.
// The other edges are BROADCAST.
private void testSchedulingWithPartitionStats(FairRoutingType fairRoutingType, int numTasks, long[] partitionStats, int numCompletedEvents, int expectedScheduledTasks, int expectedNumDestinationConsumerTasks, Map<String, EdgeManagerPlugin> newEdgeManagers) throws Exception {
    Configuration conf = new Configuration();
    FairShuffleVertexManager manager;
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String r1 = "R1";
    final int numOfTasksInr1 = numTasks;
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String m2 = "M2";
    final int numOfTasksInM2 = 3;
    EdgeProperty eProp2 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String m3 = "M3";
    final int numOfTasksInM3 = 3;
    EdgeProperty eProp3 = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    final String mockManagedVertexId = "R2";
    final int numOfTasksInDestination = 3;
    mockInputVertices.put(r1, eProp1);
    mockInputVertices.put(m2, eProp2);
    mockInputVertices.put(m3, eProp3);
    final VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(numOfTasksInDestination);
    when(mockContext.getVertexNumTasks(r1)).thenReturn(numOfTasksInr1);
    when(mockContext.getVertexNumTasks(m2)).thenReturn(numOfTasksInM2);
    when(mockContext.getVertexNumTasks(m3)).thenReturn(numOfTasksInM3);
    final List<Integer> scheduledTasks = Lists.newLinkedList();
    doAnswer(new ScheduledTasksAnswer(scheduledTasks)).when(mockContext).scheduleTasks(anyList());
    doAnswer(new reconfigVertexAnswer(mockContext, mockManagedVertexId, newEdgeManagers)).when(mockContext).reconfigureVertex(anyInt(), any(VertexLocationHint.class), anyMap());
    // check initialization
    manager = createFairShuffleVertexManager(conf, mockContext, fairRoutingType, 1000 * MB, 0.001f, 0.001f);
    manager.onVertexStarted(emptyCompletions);
    Assert.assertTrue(manager.bipartiteSources == 1);
    manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(m2, VertexState.CONFIGURED));
    Assert.assertEquals(numOfTasksInDestination, // no tasks scheduled
    manager.pendingTasks.size());
    Assert.assertEquals(numOfTasksInr1, manager.totalNumBipartiteSourceTasks);
    Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
    // no tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == numOfTasksInDestination);
    Assert.assertTrue(manager.totalNumBipartiteSourceTasks == numOfTasksInr1);
    for (int i = 0; i < numCompletedEvents; i++) {
        VertexManagerEvent vmEvent = getVertexManagerEvent(partitionStats, 0, r1, true);
        manager.onSourceTaskCompleted(vmEvent.getProducerAttemptIdentifier());
        // send VM event
        manager.onVertexManagerEventReceived(vmEvent);
    }
    // Send an event for m2.
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(m2, 0));
    // no tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == numOfTasksInDestination);
    Assert.assertTrue(manager.totalNumBipartiteSourceTasks == numOfTasksInr1);
    // Send an event for m3.
    manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
    manager.onSourceTaskCompleted(createTaskAttemptIdentifier(m3, 0));
    // all tasks scheduled
    Assert.assertTrue(manager.pendingTasks.size() == 0);
    Assert.assertTrue(scheduledTasks.size() == expectedScheduledTasks);
    Assert.assertEquals(1, newEdgeManagers.size());
    EdgeManagerPluginOnDemand edgeManager = (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next();
    // the same as original number of partitions.
    for (int i = 0; i < numOfTasksInr1; i++) {
        Assert.assertEquals(numOfTasksInDestination, edgeManager.getNumSourceTaskPhysicalOutputs(0));
    }
    for (int sourceTaskIndex = 0; sourceTaskIndex < numOfTasksInr1; sourceTaskIndex++) {
        Assert.assertEquals(expectedNumDestinationConsumerTasks, edgeManager.getNumDestinationConsumerTasks(sourceTaskIndex));
    }
}

Example 49 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestInputReadyVertexManager method testBasicScatterGather.

@Test(timeout = 5000)
public void testBasicScatterGather() throws Exception {
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockManagedVertexId = "Vertex";
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(2);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    InputReadyVertexManager manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(1)).vertexReconfigurationPlanned();
    // source vertex configured
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    verify(mockContext, times(1)).doneReconfiguringVertex();
    verify(mockContext, times(0)).scheduleTasks(requestCaptor.capture());
    // then own vertex started
    manager.onVertexStarted(Collections.singletonList(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0)));
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    verify(mockContext, times(0)).scheduleTasks(anyList());
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    verify(mockContext, times(1)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(2, requestCaptor.getValue().size());
}

Example 50 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestInputReadyVertexManager method testBasicOneToOne.

@Test(timeout = 5000)
public void testBasicOneToOne() throws Exception {
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockManagedVertexId = "Vertex";
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    InputReadyVertexManager manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(1)).vertexReconfigurationPlanned();
    // source vertex configured
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    verify(mockContext, times(1)).doneReconfiguringVertex();
    verify(mockContext, times(0)).scheduleTasks(requestCaptor.capture());
    manager.onVertexStarted(Collections.singletonList(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0)));
    verify(mockContext, times(1)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    verify(mockContext, times(2)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    verify(mockContext, times(3)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
}