Examples with TezTaskAttemptID org.apache.tez.dag.records.TezTaskAttemptID used on opensource projects

Example 91 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project tez by apache.

the class TestVertexImpl method testSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testSourceTaskAttemptCompletionEvents() {
    LOG.info("Testing testSourceTaskAttemptCompletionEvents");
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta2_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TezTaskAttemptID ta2_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    v4.handle(new VertexEventTaskAttemptCompleted(ta1_t1_v4, TaskAttemptStateInternal.FAILED));
    v4.handle(new VertexEventTaskAttemptCompleted(ta2_t1_v4, TaskAttemptStateInternal.SUCCEEDED));
    v4.handle(new VertexEventTaskAttemptCompleted(ta1_t2_v4, TaskAttemptStateInternal.SUCCEEDED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta1_t1_v5, TaskAttemptStateInternal.SUCCEEDED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta1_t2_v5, TaskAttemptStateInternal.FAILED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta2_t2_v5, TaskAttemptStateInternal.SUCCEEDED));
    v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
    v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
    v5.handle(new VertexEventTaskCompleted(t1_v5, TaskState.SUCCEEDED));
    v5.handle(new VertexEventTaskCompleted(t2_v5, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(4, v6.numSuccessSourceAttemptCompletions);
}

Example 92 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project tez by apache.

the class TestVertexImpl method testPickupDagLocalResourceOnScheduleTask.

@Test(timeout = 5000)
public void testPickupDagLocalResourceOnScheduleTask() {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    TezTaskAttemptID taskAttemptId0 = TezTaskAttemptID.getInstance(v1.getTask(0).getTaskId(), 0);
    TaskAttemptImpl ta0 = (TaskAttemptImpl) v1.getTask(0).getAttempt(taskAttemptId0);
    ta0.handle(new TaskAttemptEventSchedule(taskAttemptId0, 1, 1));
    dispatcher.await();
    Assert.assertEquals(1, amSchedulerEventDispatcher.events.size());
    AMSchedulerEventTALaunchRequest launchRequestEvent = (AMSchedulerEventTALaunchRequest) amSchedulerEventDispatcher.events.get(0);
    Map<String, LocalResource> localResourceMap = launchRequestEvent.getContainerContext().getLocalResources();
    Assert.assertTrue(localResourceMap.containsKey("dag lr"));
    Assert.assertTrue(localResourceMap.containsKey("vertex lr"));
}

Example 93 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project tez by apache.

the class TestVertexImpl method testVertexGetTAAttemptsObsoletion.

@Test(timeout = 5000)
public void testVertexGetTAAttemptsObsoletion() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    VertexImpl v2 = vertices.get("vertex2");
    startVertex(v2);
    VertexImpl v3 = vertices.get("vertex3");
    VertexImpl v4 = vertices.get("vertex4");
    List<ScheduleTaskRequest> taskList = new LinkedList<VertexManagerPluginContext.ScheduleTaskRequest>();
    // scheduling start to trigger edge routing to begin
    for (int i = 0; i < v4.getTotalTasks(); ++i) {
        taskList.add(ScheduleTaskRequest.create(i, null));
    }
    v4.scheduleTasks(taskList);
    Assert.assertEquals(VertexState.RUNNING, v4.getState());
    Assert.assertEquals(1, v4.sourceVertices.size());
    Edge e = v4.sourceVertices.get(v3);
    TezTaskAttemptID v3TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v3.getVertexId(), 0), 0);
    TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v4.getVertexId(), 0), 0);
    for (int i = 0; i < 11; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // verify all events have been are in taskEvents
    Assert.assertEquals(11, v4.getOnDemandRouteEvents().size());
    TaskAttemptEventInfo eventInfo;
    EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class);
    EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[] { 0 });
    e.edgeManager = mockPlugin;
    when(mockPlugin.routeInputSourceTaskFailedEventToDestination(anyInt(), anyInt())).thenReturn(mockRoute);
    when(mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute);
    // send an input failed event
    v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(InputFailedEvent.create(0, 0), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    // ask for events with sufficient buffer. get only input failed event. all DM events obsoleted
    int fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    fromEventId = eventInfo.getNextFromEventId();
    Assert.assertEquals(12, fromEventId);
    Assert.assertEquals(1, eventInfo.getEvents().size());
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
    // Let failed task send more event
    for (int i = 11; i < 14; ++i) {
        v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList(new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId)))));
    }
    dispatcher.await();
    // 11 events + 1 INPUT_FAILED_EVENT.
    // Events sent out later by failed tasks should not be available.
    Assert.assertEquals(12, v4.getOnDemandRouteEvents().size());
    fromEventId = 0;
    eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100);
    Assert.assertEquals(EventType.INPUT_FAILED_EVENT, eventInfo.getEvents().get(0).getEventType());
}

Example 94 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project tez by apache.

the class TestVertexImpl method testRouteEvent_RecoveredEvent.

@Test(timeout = 5000)
public void testRouteEvent_RecoveredEvent() throws IOException {
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(true).when(appContext).isRecoveryEnabled();
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = (VertexImpl) vertices.get("vertex1");
    VertexImpl v2 = (VertexImpl) vertices.get("vertex2");
    VertexImpl v3 = (VertexImpl) vertices.get("vertex3");
    startVertex(v1);
    startVertex(v2);
    TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0);
    v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
    DataMovementEvent dmEvent = DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(taskId, 0);
    TezEvent tezEvent1 = new TezEvent(dmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent1)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() != 0);
    // ArgumentCaptor<DAGHistoryEvent> argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    // verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
    // verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
    v3.scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() == 0);
// recovery events is not only handled one time
// argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
}

Example 95 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project tez by apache.

the class TestVertexImpl method sendTaskGeneratedEvent.

private void sendTaskGeneratedEvent(Event event, EventProducerConsumerType generator, Vertex taskVertex, Vertex edgeVertex) {
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(taskVertex.getVertexId(), 1), 1);
    EventMetaData sourceInfo = new EventMetaData(generator, taskVertex.getName(), edgeVertex.getName(), taId);
    sendVertexEventRouteEvent(taskVertex, new TezEvent(event, sourceInfo));
}