Examples with Task org.apache.tez.dag.app.dag.Task used on opensource projects

Example 11 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestVertexImpl method checkTasks.

private void checkTasks(Vertex v, int numTasks) {
    Assert.assertEquals(numTasks, v.getTotalTasks());
    Map<TezTaskID, Task> tasks = v.getTasks();
    Assert.assertEquals(numTasks, tasks.size());
    // check all indices
    int i = 0;
    // iteration maintains order due to linked hash map
    for (Task task : tasks.values()) {
        Assert.assertEquals(i, task.getTaskId().getId());
        i++;
    }
}

Example 12 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestVertexImpl method testTaskSchedulingWithCustomEdges.

@Test(timeout = 5000)
public /**
 * Ref: TEZ-1494
 * If broadcast, one-to-one or custom edges are present in source, tasks should not start until
 * 1 task from each source vertex is complete.
 */
void testTaskSchedulingWithCustomEdges() throws TezException {
    setupPreDagCreation();
    dagPlan = createCustomDAGWithCustomEdges();
    setupPostDagCreation();
    /**
     *   M2 --(SG)--> R3 --(B)--\
     *                           \
     *   M7 --(B)---------------->M5 ---(SG)--> R6
     *                            /
     *   M8---(C)--------------->/
     *                          /
     *   M9---(B)--------------> (zero task vertex)
     */
    // init M2
    VertexImpl m2 = vertices.get("M2");
    VertexImpl m7 = vertices.get("M7");
    VertexImpl r3 = vertices.get("R3");
    VertexImpl m5 = vertices.get("M5");
    VertexImpl m8 = vertices.get("M8");
    VertexImpl m9 = vertices.get("M9");
    initVertex(m2);
    initVertex(m7);
    initVertex(m8);
    initVertex(m9);
    assertTrue(m7.getState().equals(VertexState.INITED));
    assertTrue(m9.getState().equals(VertexState.INITED));
    assertTrue(m5.getState().equals(VertexState.INITED));
    assertTrue(m8.getState().equals(VertexState.INITED));
    assertTrue(m5.getVertexManager().getPlugin() instanceof ImmediateStartVertexManager);
    // Start M9
    dispatcher.getEventHandler().handle(new VertexEvent(m9.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // Start M2; Let tasks complete in M2; Also let 1 task complete in R3
    dispatcher.getEventHandler().handle(new VertexEvent(m2.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // R3 should be in running state
    assertTrue(r3.getState().equals(VertexState.RUNNING));
    // Let us start M7; M5 should start not start as it is dependent on M8 as well
    dispatcher.getEventHandler().handle(new VertexEvent(m7.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    // M5 should be in INITED state, as it depends on M8
    assertTrue(m5.getState().equals(VertexState.INITED));
    for (Task task : m5.getTasks().values()) {
        assertTrue(task.getState().equals(TaskState.NEW));
    }
    // Let us start M8; M5 should start now
    dispatcher.getEventHandler().handle(new VertexEvent(m8.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    assertTrue(m9.getState().equals(VertexState.SUCCEEDED));
    // M5 in running state. All source vertices have started and are configured
    assertTrue(m5.getState().equals(VertexState.RUNNING));
    for (Task task : m5.getTasks().values()) {
        assertTrue(task.getState().equals(TaskState.SCHEDULED));
    }
}

Example 13 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting() {
    // Remove after legacy routing is removed
    setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    InputFailedEvent ifEvent = InputFailedEvent.create(0, 1);
    TezEvent tezEvent = new TezEvent(ifEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name()));
}

Example 14 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestEdge method verifyEvents.

private void verifyEvents(TezTaskAttemptID srcTAID, LinkedHashMap<TezTaskID, Task> destTasks) {
    int count = 0;
    for (Entry<TezTaskID, Task> taskEntry : destTasks.entrySet()) {
        Task mockTask = taskEntry.getValue();
        ArgumentCaptor<TezEvent> args = ArgumentCaptor.forClass(TezEvent.class);
        verify(mockTask, times(1)).registerTezEvent(args.capture());
        TezEvent capturedEvent = args.getValue();
        DataMovementEvent dmEvent = (DataMovementEvent) capturedEvent.getEvent();
        assertEquals(srcTAID.getId(), dmEvent.getVersion());
        assertEquals(count++, dmEvent.getSourceIndex());
        assertEquals(srcTAID.getTaskID().getId(), dmEvent.getTargetIndex());
        byte[] res = new byte[dmEvent.getUserPayload().limit() - dmEvent.getUserPayload().position()];
        dmEvent.getUserPayload().slice().get(res);
        assertTrue(Arrays.equals("bytes".getBytes(), res));
    }
}

Example 15 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestEdge method mockTasks.

private LinkedHashMap<TezTaskID, Task> mockTasks(TezVertexID vertexID, int numTasks) {
    LinkedHashMap<TezTaskID, Task> tasks = new LinkedHashMap<TezTaskID, Task>();
    for (int i = 0; i < numTasks; i++) {
        Task task = mock(Task.class);
        TezTaskID taskID = TezTaskID.getInstance(vertexID, i);
        doReturn(taskID).when(task).getTaskId();
        tasks.put(taskID, task);
    }
    return tasks;
}