Examples with TaskAttempt org.apache.tez.dag.app.dag.TaskAttempt used on opensource projects

Example 16 with TaskAttempt

use of org.apache.tez.dag.app.dag.TaskAttempt in project tez by apache.

the class TestAMWebController method getAttemptsTestHelper.

Map<String, Object> getAttemptsTestHelper(List<TaskAttempt> attempts, List<List<Integer>> attemptMinIds, List<Integer> vertexMinIds, List<Integer> taskMinIds, Integer limit) {
    // Creating mock DAG
    DAG mockDAG = mock(DAG.class);
    doReturn(TezDAGID.fromString("dag_1441301219877_0109_1")).when(mockDAG).getID();
    // Creating mock vertex and attaching to mock DAG
    TezVertexID vertexID = TezVertexID.fromString("vertex_1441301219877_0109_1_00");
    Vertex mockVertex = mock(Vertex.class);
    doReturn(vertexID).when(mockVertex).getVertexId();
    doReturn(mockVertex).when(mockDAG).getVertex(vertexID);
    doReturn(ImmutableMap.of(vertexID, mockVertex)).when(mockDAG).getVertices();
    // Creating mock task and attaching to mock Vertex
    TezTaskID taskID = TezTaskID.fromString("task_1441301219877_0109_1_00_000000");
    Task mockTask = mock(Task.class);
    doReturn(taskID).when(mockTask).getTaskId();
    int taskIndex = taskID.getId();
    doReturn(mockTask).when(mockVertex).getTask(taskIndex);
    doReturn(ImmutableMap.of(taskID, mockTask)).when(mockVertex).getTasks();
    // Creating mock tasks and attaching to mock vertex
    Map<TezTaskAttemptID, TaskAttempt> attemptsMap = Maps.newHashMap();
    for (TaskAttempt attempt : attempts) {
        TezTaskAttemptID attemptId = attempt.getID();
        doReturn(attempt).when(mockTask).getAttempt(attemptId);
        attemptsMap.put(attemptId, attempt);
    }
    doReturn(attemptsMap).when(mockTask).getAttempts();
    // Creates & setup controller spy
    AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL");
    AMWebController spy = spy(amWebController);
    doReturn(true).when(spy).setupResponse();
    doNothing().when(spy).renderJSON(any());
    // Set mock query params
    doReturn(limit).when(spy).getQueryParamInt(WebUIService.LIMIT);
    doReturn(vertexMinIds).when(spy).getIntegersFromRequest(WebUIService.VERTEX_ID, limit);
    doReturn(taskMinIds).when(spy).getIDsFromRequest(WebUIService.TASK_ID, limit, 2);
    doReturn(attemptMinIds).when(spy).getIDsFromRequest(WebUIService.ATTEMPT_ID, limit, 3);
    // Set function mocks
    doReturn(mockDAG).when(spy).checkAndGetDAGFromRequest();
    Map<String, Set<String>> counterList = new TreeMap<String, Set<String>>();
    doReturn(counterList).when(spy).getCounterListFromRequest();
    spy.getAttemptsInfo();
    verify(spy).renderJSON(returnResultCaptor.capture());
    return returnResultCaptor.getValue();
}

Example 17 with TaskAttempt

use of org.apache.tez.dag.app.dag.TaskAttempt in project tez by apache.

the class TestMockDAGAppMaster method testInternalPreemption.

@Test(timeout = 5000)
public void testInternalPreemption() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    // there is only 1 task whose first attempt will be preempted
    DAG dag = DAG.create("testInternalPreemption");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA);
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    ContainerData cData = mockLauncher.getContainers().values().iterator().next();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TaskAttempt killedTa = dagImpl.getVertex(vA.getName()).getTask(0).getAttempt(killedTaId);
    Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
    tezClient.stop();
}

Example 18 with TaskAttempt

use of org.apache.tez.dag.app.dag.TaskAttempt in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method createTaskAttempt.

private TaskAttempt createTaskAttempt(TezVertexID vertexId, int taskIdInt, int attemptIdInt) {
    TaskAttempt taskAttempt = mock(TaskAttempt.class);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, taskIdInt);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, attemptIdInt);
    doReturn(taskAttemptId).when(taskAttempt).getID();
    doReturn(vertexId).when(taskAttempt).getVertexID();
    return taskAttempt;
}

Example 19 with TaskAttempt

use of org.apache.tez.dag.app.dag.TaskAttempt in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimitWithKilledNonRunningTask.

@Test(timeout = 5000)
public void testConcurrencyLimitWithKilledNonRunningTask() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId0, 1);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(1)));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(0).getID(), // matches order
    mockEventHandler.events.get(0).getTaskAttemptID());
}

Example 20 with TaskAttempt

use of org.apache.tez.dag.app.dag.TaskAttempt in project tez by apache.

the class TestDAGScheduler method testDAGSchedulerNaturalOrder.

@Test(timeout = 5000)
public void testDAGSchedulerNaturalOrder() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    Vertex mockVertex = mock(Vertex.class);
    TaskAttempt mockAttempt = mock(TaskAttempt.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockDag.getTotalVertices()).thenReturn(4);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0).thenReturn(1).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezVertexID vId2 = TezVertexID.fromString("vertex_1436907267600_195589_1_02");
    TezVertexID vId3 = TezVertexID.fromString("vertex_1436907267600_195589_1_03");
    when(mockVertex.getVertexId()).thenReturn(vId0).thenReturn(vId1).thenReturn(vId2).thenReturn(vId3);
    DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(10, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(12, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(25, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(27, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(40, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(42, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(43, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(45, mockEventHandler.event.getPriorityLowLimit());
}