Examples with TaskAttemptIdentifier org.apache.tez.runtime.api.TaskAttemptIdentifier used on opensource projects

Example 6 with TaskAttemptIdentifier

use of org.apache.tez.runtime.api.TaskAttemptIdentifier in project tez by apache.

the class VertexManagerPlugin method onVertexStarted.

/**
 * Notification that the vertex is ready to start running tasks
 * @param completions All the source task attempts that have already completed
 * @throws Exception
 */
public void onVertexStarted(List<TaskAttemptIdentifier> completions) throws Exception {
    Map<String, List<Integer>> completionsMap = new HashMap<String, List<Integer>>();
    for (TaskAttemptIdentifier attempt : completions) {
        String vName = attempt.getTaskIdentifier().getVertexIdentifier().getName();
        List<Integer> tasks = completionsMap.get(vName);
        if (tasks == null) {
            tasks = new LinkedList<Integer>();
            completionsMap.put(vName, tasks);
        }
        tasks.add(attempt.getTaskIdentifier().getIdentifier());
    }
    onVertexStarted(completionsMap);
}

Example 7 with TaskAttemptIdentifier

use of org.apache.tez.runtime.api.TaskAttemptIdentifier in project tez by apache.

the class CartesianProductVertexManagerPartitioned method onVertexStarted.

@Override
public synchronized void onVertexStarted(List<TaskAttemptIdentifier> completions) throws Exception {
    vertexStarted = true;
    if (completions != null) {
        for (TaskAttemptIdentifier attempt : completions) {
            onSourceTaskCompleted(attempt);
        }
    }
    // try schedule because there may be no more vertex state update and source completions
    tryScheduleTask();
}

Example 8 with TaskAttemptIdentifier

use of org.apache.tez.runtime.api.TaskAttemptIdentifier in project tez by apache.

the class VertexImpl method getTaskAttemptIdentifiers.

private static List<TaskAttemptIdentifier> getTaskAttemptIdentifiers(DAG dag, List<TezTaskAttemptID> taIds) {
    List<TaskAttemptIdentifier> attempts = new ArrayList<TaskAttemptIdentifier>(taIds.size());
    String dagName = dag.getName();
    for (TezTaskAttemptID taId : taIds) {
        String vertexName = dag.getVertex(taId.getTaskID().getVertexID()).getName();
        attempts.add(getTaskAttemptIdentifier(dagName, vertexName, taId));
    }
    return attempts;
}

Example 9 with TaskAttemptIdentifier

use of org.apache.tez.runtime.api.TaskAttemptIdentifier in project tez by apache.

the class RootInputVertexManager method onVertexStarted.

@Override
public void onVertexStarted(List<TaskAttemptIdentifier> completions) {
    Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
    for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) {
        String srcVertex = entry.getKey();
        // track vertices with task count > 0
        int numTasks = getContext().getVertexNumTasks(srcVertex);
        if (numTasks > 0) {
            LOG.info("Task count in " + srcVertex + ": " + numTasks);
            srcVertexInfo.put(srcVertex, createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName())));
            getContext().registerForVertexStateUpdates(srcVertex, EnumSet.of(VertexState.CONFIGURED));
        } else {
            LOG.info("Vertex: " + getContext().getVertexName() + "; Ignoring " + srcVertex + " as it has " + numTasks + " tasks");
        }
    }
    if (completions != null) {
        for (TaskAttemptIdentifier attempt : completions) {
            onSourceTaskCompleted(attempt);
        }
    }
    onVertexStartedDone.set(true);
    // track the tasks in this vertex
    updatePendingTasks();
    processPendingTasks();
}

Example 10 with TaskAttemptIdentifier

use of org.apache.tez.runtime.api.TaskAttemptIdentifier in project tez by apache.

the class TestInputReadyVertexManager method testComplex.

@Test(timeout = 5000)
public void testComplex() throws Exception {
    HashMap<String, EdgeProperty> mockInputVertices = new HashMap<String, EdgeProperty>();
    String mockSrcVertexId1 = "Vertex1";
    EdgeProperty eProp1 = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockSrcVertexId2 = "Vertex2";
    EdgeProperty eProp2 = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockSrcVertexId3 = "Vertex3";
    EdgeProperty eProp3 = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in"));
    String mockManagedVertexId = "Vertex";
    Container mockContainer2 = mock(Container.class);
    ContainerId mockCId2 = mock(ContainerId.class);
    when(mockContainer2.getId()).thenReturn(mockCId2);
    Container mockContainer3 = mock(Container.class);
    ContainerId mockCId3 = mock(ContainerId.class);
    when(mockContainer3.getId()).thenReturn(mockCId3);
    VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
    when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
    when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
    when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(3);
    mockInputVertices.put(mockSrcVertexId1, eProp1);
    mockInputVertices.put(mockSrcVertexId2, eProp2);
    mockInputVertices.put(mockSrcVertexId3, eProp3);
    List<TaskAttemptIdentifier> initialCompletions = Lists.newArrayList();
    // 1-1 sources do not match managed tasks. setParallelism called to make them match
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(4);
    InputReadyVertexManager manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(1)).vertexReconfigurationPlanned();
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
    verify(mockContext, times(1)).reconfigureVertex(3, null, null);
    verify(mockContext, times(1)).doneReconfiguringVertex();
    manager.onVertexStarted(initialCompletions);
    // 1-1 sources do not match
    when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
    when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(4);
    manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(2)).vertexReconfigurationPlanned();
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    try {
        manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
        Assert.assertTrue("Should have exception", false);
    } catch (TezUncheckedException e) {
        e.getMessage().contains("1-1 source vertices must have identical concurrency");
    }
    verify(mockContext, times(1)).reconfigureVertex(anyInt(), (VertexLocationHint) any(), // not invoked
    anyMap());
    when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(3);
    initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0));
    initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId2, 0));
    manager = new InputReadyVertexManager(mockContext);
    manager.initialize();
    verify(mockContext, times(3)).vertexReconfigurationPlanned();
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
    manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
    verify(mockContext, times(1)).reconfigureVertex(anyInt(), (VertexLocationHint) any(), // not invoked
    anyMap());
    verify(mockContext, times(2)).doneReconfiguringVertex();
    manager.onVertexStarted(initialCompletions);
    // all 1-1 0's done but not scheduled because v1 is not done
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 0));
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    manager.onSourceTaskCompleted(// duplicate
    TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 1));
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId2, 1));
    verify(mockContext, times(0)).scheduleTasks(anyList());
    manager.onSourceTaskCompleted(// v1 done
    TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2));
    verify(mockContext, times(1)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId3, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
    // 1-1 completion triggers since other 1-1 is done
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 1));
    verify(mockContext, times(2)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId3, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
    // 1-1 completion does not trigger since other 1-1 is not done
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 2));
    verify(mockContext, times(2)).scheduleTasks(anyList());
    // 1-1 completion trigger start
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId2, 2));
    verify(mockContext, times(3)).scheduleTasks(requestCaptor.capture());
    Assert.assertEquals(1, requestCaptor.getValue().size());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskIndex());
    Assert.assertEquals(mockSrcVertexId2, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getVertexName());
    Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskLocationHint().getAffinitizedTask().getTaskIndex());
    // no more starts
    manager.onSourceTaskCompleted(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 2));
    verify(mockContext, times(3)).scheduleTasks(anyList());
}