Examples with TaskAttemptEventSchedule org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule used on opensource projects

Example 31 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestTaskImpl method testSucceededAttemptStatusWithRetroActiveFailures.

@SuppressWarnings("rawtypes")
@Test
public void testSucceededAttemptStatusWithRetroActiveFailures() throws InterruptedException {
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0);
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getAttemptList().get(1);
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    firstMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getID()));
    mockTask.handle(new TaskEventTASucceeded(firstMockTaskAttempt.getID()));
    assertTrue("Attempts should have succeeded!", firstMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED);
    assertEquals("Task should have no uncompleted attempts!", 0, mockTask.getUncompletedAttemptsCount());
    assertTrue("Task should have Succeeded!", mockTask.getState() == TaskState.SUCCEEDED);
    // Failing the attempt that finished after the task was marked succeeded, should not schedule another attempt
    failAttempt(firstMockTaskAttempt, 0, 0);
    assertTaskSucceededState();
    // Failing the attempt that allowed the task to succeed, should schedule another attempt
    failAttempt(secondMockTaskAttempt, 1, 1);
    assertTaskScheduledState();
}

Example 32 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class DAGSchedulerNaturalOrderControlled method scheduleTaskEx.

// TODO Does ordering matter - it currently depends on the order returned by vertex.getOutput*
@Override
public void scheduleTaskEx(DAGEventSchedulerUpdate event) {
    TaskAttempt attempt = event.getAttempt();
    Vertex vertex = dag.getVertex(attempt.getVertexID());
    int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
    // natural priority. Handles failures and retries.
    int priorityLowLimit = ((vertexDistanceFromRoot + 1) * dag.getTotalVertices() * 3) + (vertex.getVertexId().getId() * 3);
    int priorityHighLimit = priorityLowLimit - 2;
    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(attempt.getID(), priorityLowLimit, priorityHighLimit);
    taskAttemptSeen(vertex.getName(), attempt.getID());
    if (vertexAlreadyScheduled(vertex)) {
        // Vertex previously marked ready for scheduling.
        if (LOG.isDebugEnabled()) {
            LOG.debug("Scheduling " + attempt.getID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit);
        }
        sendEvent(attemptEvent);
        // A new taks coming in here could send us over the enough tasks scheduled limit.
        processDownstreamVertices(vertex);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Attempting to schedule vertex: " + vertex.getLogIdentifier() + " due to schedule event");
        }
        boolean scheduled = trySchedulingVertex(vertex);
        if (scheduled) {
            LOG.info("Scheduled vertex: " + vertex.getLogIdentifier());
            // If ready to be scheduled, send out pending events and the current event.
            // Send events out for this vertex first. Then try scheduling downstream vertices.
            sendEventsForVertex(vertex.getName());
            sendEvent(attemptEvent);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Processing downstream vertices for vertex: " + vertex.getLogIdentifier());
            }
            processDownstreamVertices(vertex);
        } else {
            pendingEvents.put(vertex.getName(), attemptEvent);
        }
    }
}

Example 33 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class DAGSchedulerNaturalOrder method scheduleTaskEx.

@Override
public void scheduleTaskEx(DAGEventSchedulerUpdate event) {
    TaskAttempt attempt = event.getAttempt();
    Vertex vertex = dag.getVertex(attempt.getVertexID());
    int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
    // natural priority. Handles failures and retries.
    int priorityLowLimit = ((vertexDistanceFromRoot + 1) * dag.getTotalVertices() * 3) + (vertex.getVertexId().getId() * 3);
    int priorityHighLimit = priorityLowLimit - 2;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scheduling " + attempt.getID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit);
    }
    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(attempt.getID(), priorityLowLimit, priorityHighLimit);
    sendEvent(attemptEvent);
}