Examples with TaskType org.apache.hadoop.mapreduce.TaskType used on opensource projects

Example 1 with TaskType

use of org.apache.hadoop.mapreduce.TaskType in project hadoop by apache.

the class ZombieJob method getMapTaskAttemptInfoAdjusted.

/**
   * Get a {@link TaskAttemptInfo} with a {@link TaskAttemptID} associated with
   * taskType, taskNumber, and taskAttemptNumber. This function considers
   * locality, and follows the following decision logic: 1. Make up a
   * {@link TaskAttemptInfo} if the task attempt is missing in trace, 2. Make up
   * a {@link TaskAttemptInfo} if the task attempt has a KILLED final status in
   * trace, 3. If final state is FAILED, construct a {@link TaskAttemptInfo}
   * from the trace, without considering locality. 4. If final state is
   * SUCCEEDED, construct a {@link TaskAttemptInfo} from the trace, with runtime
   * scaled according to locality in simulation and locality in trace.
   */
@Override
public TaskAttemptInfo getMapTaskAttemptInfoAdjusted(int taskNumber, int taskAttemptNumber, int locality) {
    TaskType taskType = TaskType.MAP;
    LoggedTask loggedTask = getLoggedTask(taskType, taskNumber);
    if (loggedTask == null) {
        // TODO insert parameters
        TaskInfo taskInfo = new TaskInfo(0, 0, 0, 0, 0);
        return makeUpTaskAttemptInfo(taskType, taskInfo, taskAttemptNumber, taskNumber, locality);
    }
    LoggedTaskAttempt loggedAttempt = getLoggedTaskAttempt(taskType, taskNumber, taskAttemptNumber);
    if (loggedAttempt == null) {
        // Task exists, but attempt is missing.
        TaskInfo taskInfo = getTaskInfo(loggedTask);
        return makeUpTaskAttemptInfo(taskType, taskInfo, taskAttemptNumber, taskNumber, locality);
    } else {
        // Task and TaskAttempt both exist.
        if (loggedAttempt.getResult() == Values.KILLED) {
            TaskInfo taskInfo = getTaskInfo(loggedTask);
            return makeUpTaskAttemptInfo(taskType, taskInfo, taskAttemptNumber, taskNumber, locality);
        } else if (loggedAttempt.getResult() == Values.FAILED) {
            /**
         * FAILED attempt is not affected by locality however, made-up FAILED
         * attempts ARE affected by locality, since statistics are present for
         * attempts of different locality.
         */
            return getTaskAttemptInfo(loggedTask, loggedAttempt);
        } else if (loggedAttempt.getResult() == Values.SUCCESS) {
            int loggedLocality = getLocality(loggedTask, loggedAttempt);
            if (locality == loggedLocality) {
                return getTaskAttemptInfo(loggedTask, loggedAttempt);
            } else {
                // a different locality.
                return scaleInfo(loggedTask, loggedAttempt, locality, loggedLocality, rackLocalOverNodeLocal, rackRemoteOverNodeLocal);
            }
        } else {
            throw new IllegalArgumentException("attempt result is not SUCCEEDED, FAILED or KILLED: " + loggedAttempt.getResult());
        }
    }
}

Example 2 with TaskType

use of org.apache.hadoop.mapreduce.TaskType in project hadoop by apache.

the class TestRecovery method getMockTaskAttemptInfo.

private TaskAttemptInfo getMockTaskAttemptInfo(TaskAttemptID tai, TaskAttemptState tas) {
    ContainerId ci = mock(ContainerId.class);
    Counters counters = mock(Counters.class);
    TaskType tt = TaskType.MAP;
    long finishTime = System.currentTimeMillis();
    TaskAttemptInfo mockTAinfo = mock(TaskAttemptInfo.class);
    when(mockTAinfo.getAttemptId()).thenReturn(tai);
    when(mockTAinfo.getContainerId()).thenReturn(ci);
    when(mockTAinfo.getCounters()).thenReturn(counters);
    when(mockTAinfo.getError()).thenReturn("");
    when(mockTAinfo.getFinishTime()).thenReturn(finishTime);
    when(mockTAinfo.getHostname()).thenReturn("localhost");
    when(mockTAinfo.getHttpPort()).thenReturn(23);
    when(mockTAinfo.getMapFinishTime()).thenReturn(finishTime - 1000L);
    when(mockTAinfo.getPort()).thenReturn(24);
    when(mockTAinfo.getRackname()).thenReturn("defaultRack");
    when(mockTAinfo.getShuffleFinishTime()).thenReturn(finishTime - 2000L);
    when(mockTAinfo.getShufflePort()).thenReturn(25);
    when(mockTAinfo.getSortFinishTime()).thenReturn(finishTime - 3000L);
    when(mockTAinfo.getStartTime()).thenReturn(finishTime - 10000);
    when(mockTAinfo.getState()).thenReturn("task in progress");
    when(mockTAinfo.getTaskStatus()).thenReturn(tas.toString());
    when(mockTAinfo.getTaskType()).thenReturn(tt);
    when(mockTAinfo.getTrackerName()).thenReturn("TrackerName");
    return mockTAinfo;
}

Example 3 with TaskType

use of org.apache.hadoop.mapreduce.TaskType in project hadoop by apache.

the class JobFactory method getNextJobFiltered.

protected JobStory getNextJobFiltered() throws IOException {
    JobStory job = getNextJobFromTrace();
    // These jobs are not yet supported in Gridmix
    while (job != null && (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS || job.getSubmissionTime() < 0 || job.getNumberMaps() == 0)) {
        if (LOG.isDebugEnabled()) {
            List<String> reason = new ArrayList<String>();
            if (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS) {
                reason.add("STATE (" + job.getOutcome().name() + ")");
            }
            if (job.getSubmissionTime() < 0) {
                reason.add("SUBMISSION-TIME (" + job.getSubmissionTime() + ")");
            }
            if (job.getNumberMaps() == 0) {
                reason.add("ZERO-MAPS-JOB");
            }
            // TODO This should never happen. Probably we missed something!
            if (reason.size() == 0) {
                reason.add("N/A");
            }
            LOG.debug("Ignoring job " + job.getJobID() + " from the input trace." + " Reason: " + StringUtils.join(reason, ","));
        }
        job = getNextJobFromTrace();
    }
    return null == job ? null : new FilterJobStory(job) {

        @Override
        public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
            TaskInfo info = this.job.getTaskInfo(taskType, taskNumber);
            if (info != null) {
                info = new MinTaskInfo(info);
            } else {
                info = new MinTaskInfo(new TaskInfo(0, 0, 0, 0, 0));
            }
            return info;
        }
    };
}

Example 4 with TaskType

use of org.apache.hadoop.mapreduce.TaskType in project hadoop by apache.

the class ZombieJob method maskTaskID.

/**
   * Mask the job ID part in a {@link TaskID}.
   * 
   * @param taskId
   *          raw {@link TaskID} read from trace
   * @return masked {@link TaskID} with empty {@link JobID}.
   */
private TaskID maskTaskID(TaskID taskId) {
    JobID jobId = new JobID();
    TaskType taskType = taskId.getTaskType();
    return new TaskID(jobId, taskType, taskId.getId());
}

Example 5 with TaskType

use of org.apache.hadoop.mapreduce.TaskType in project hadoop by apache.

the class ZombieJob method maskAttemptID.

/**
   * Mask the job ID part in a {@link TaskAttemptID}.
   * 
   * @param attemptId
   *          raw {@link TaskAttemptID} read from trace
   * @return masked {@link TaskAttemptID} with empty {@link JobID}.
   */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
    JobID jobId = new JobID();
    TaskType taskType = attemptId.getTaskType();
    TaskID taskId = attemptId.getTaskID();
    return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType, taskId.getId(), attemptId.getId());
}