Examples with DAGEventSchedulerUpdate org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate used on opensource projects

Example 1 with DAGEventSchedulerUpdate

use of org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimit.

@Test(timeout = 5000)
public void testConcurrencyLimit() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TezTaskID tId1 = TezTaskID.getInstance(vId1, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    // not effective
    scheduler.addVertexConcurrencyLimit(vId0, 0);
    // schedule beyond limit and it gets scheduled
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 0));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(1, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 1));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(2, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 2));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(3, mockEventHandler.events.size());
    mockEventHandler.events.clear();
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId1, 2);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // no extra scheduling
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
}

Example 2 with DAGEventSchedulerUpdate

use of org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate in project tez by apache.

the class DAGScheduler method taskCompleted.

public void taskCompleted(DAGEventSchedulerUpdate event) {
    taskCompletedEx(event);
    if (vertexInfo != null) {
        VertexInfo vInfo = vertexInfo.get(event.getAttempt().getID().getTaskID().getVertexID());
        if (vInfo != null) {
            if (vInfo.pendingAttempts.remove(event.getAttempt().getID()) == null) {
                vInfo.concurrency--;
                if (!vInfo.pendingAttempts.isEmpty()) {
                    Iterator<DAGEventSchedulerUpdate> i = vInfo.pendingAttempts.values().iterator();
                    DAGEventSchedulerUpdate nextTaskAttempt = i.next();
                    i.remove();
                    scheduleTaskWithLimit(nextTaskAttempt, vInfo);
                }
            }
        }
    }
}

Example 3 with DAGEventSchedulerUpdate

use of org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimitWithKilledNonRunningTask.

@Test(timeout = 5000)
public void testConcurrencyLimitWithKilledNonRunningTask() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId0, 1);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(1)));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(0).getID(), // matches order
    mockEventHandler.events.get(0).getTaskAttemptID());
}

Example 4 with DAGEventSchedulerUpdate

use of org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate in project tez by apache.

the class TestDAGScheduler method testDAGSchedulerNaturalOrder.

@Test(timeout = 5000)
public void testDAGSchedulerNaturalOrder() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    Vertex mockVertex = mock(Vertex.class);
    TaskAttempt mockAttempt = mock(TaskAttempt.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockDag.getTotalVertices()).thenReturn(4);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0).thenReturn(1).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezVertexID vId2 = TezVertexID.fromString("vertex_1436907267600_195589_1_02");
    TezVertexID vId3 = TezVertexID.fromString("vertex_1436907267600_195589_1_03");
    when(mockVertex.getVertexId()).thenReturn(vId0).thenReturn(vId1).thenReturn(vId2).thenReturn(vId3);
    DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(10, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(12, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(25, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(27, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(40, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(42, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(43, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(45, mockEventHandler.event.getPriorityLowLimit());
}

Example 5 with DAGEventSchedulerUpdate

use of org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate in project tez by apache.

the class TaskImpl method addAndScheduleAttempt.

// This is always called in the Write Lock
private void addAndScheduleAttempt(TezTaskAttemptID schedulingCausalTA) {
    TaskAttempt attempt = createAttempt(attempts.size(), schedulingCausalTA);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Created attempt " + attempt.getID());
    }
    switch(attempts.size()) {
        case 0:
            attempts = Collections.singletonMap(attempt.getID(), attempt);
            break;
        case 1:
            Map<TezTaskAttemptID, TaskAttempt> newAttempts = new LinkedHashMap<TezTaskAttemptID, TaskAttempt>(maxFailedAttempts);
            newAttempts.putAll(attempts);
            attempts = newAttempts;
            Preconditions.checkArgument(attempts.put(attempt.getID(), attempt) == null, attempt.getID() + " already existed");
            break;
        default:
            Preconditions.checkArgument(attempts.put(attempt.getID(), attempt) == null, attempt.getID() + " already existed");
            break;
    }
    // TODO: Recovery
    /*
    // Update nextATtemptNumber
    if (taskAttemptsFromPreviousGeneration.isEmpty()) {
      ++nextAttemptNumber;
    } else {
      // There are still some TaskAttempts from previous generation, use them
      nextAttemptNumber =
          taskAttemptsFromPreviousGeneration.remove(0).getAttemptId().getId();
    }
    */
    this.taskAttemptStatus.put(attempt.getID().getId(), false);
    // schedule the nextAttemptNumber
    // send event to DAG to assign priority and schedule the attempt with global
    // picture in mind
    eventHandler.handle(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, attempt));
}