Examples with TaskType org.apache.hadoop.mapreduce.v2.api.records.TaskType used on opensource projects

Example 16 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class TaskAttemptImpl method updateMillisCounters.

private static void updateMillisCounters(JobCounterUpdateEvent jce, TaskAttemptImpl taskAttempt) {
    // if container/resource if not allocated, do not update
    if (null == taskAttempt.container || null == taskAttempt.container.getResource()) {
        return;
    }
    long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
    Resource allocatedResource = taskAttempt.container.getResource();
    int mbAllocated = (int) allocatedResource.getMemorySize();
    int vcoresAllocated = allocatedResource.getVirtualCores();
    int minSlotMemSize = taskAttempt.conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
    int simSlotsAllocated = minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbAllocated / minSlotMemSize);
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    if (taskType == TaskType.MAP) {
        jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsAllocated * duration);
        jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbAllocated);
        jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresAllocated);
        jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
    } else {
        jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsAllocated * duration);
        jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbAllocated);
        jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresAllocated);
        jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
    }
}

Example 17 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class TaskAttemptImpl method createJobCounterUpdateEventTAKilled.

private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
    if (taskType == TaskType.MAP) {
        jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
    } else {
        jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
    }
    if (!taskAlreadyCompleted) {
        updateMillisCounters(jce, taskAttempt);
    }
    return jce;
}

Example 18 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class JobImpl method getTasks.

@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
    Map<TaskId, Task> localTasksCopy = tasks;
    Map<TaskId, Task> result = new HashMap<TaskId, Task>();
    Set<TaskId> tasksOfGivenType = null;
    readLock.lock();
    try {
        if (TaskType.MAP == taskType) {
            tasksOfGivenType = mapTasks;
        } else {
            tasksOfGivenType = reduceTasks;
        }
        for (TaskId taskID : tasksOfGivenType) result.put(taskID, localTasksCopy.get(taskID));
        return result;
    } finally {
        readLock.unlock();
    }
}

Example 19 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class MockJobs method newTask.

public static Task newTask(JobId jid, int i, int m, final boolean hasFailedTasks) {
    final TaskId tid = Records.newRecord(TaskId.class);
    tid.setJobId(jid);
    tid.setId(i);
    tid.setTaskType(TASK_TYPES.next());
    final TaskReport report = newTaskReport(tid);
    final Map<TaskAttemptId, TaskAttempt> attempts = newTaskAttempts(tid, m);
    return new Task() {

        @Override
        public TaskId getID() {
            return tid;
        }

        @Override
        public TaskReport getReport() {
            return report;
        }

        @Override
        public Counters getCounters() {
            if (hasFailedTasks) {
                return null;
            }
            return new Counters(TypeConverter.fromYarn(report.getCounters()));
        }

        @Override
        public float getProgress() {
            return report.getProgress();
        }

        @Override
        public TaskType getType() {
            return tid.getTaskType();
        }

        @Override
        public Map<TaskAttemptId, TaskAttempt> getAttempts() {
            return attempts;
        }

        @Override
        public TaskAttempt getAttempt(TaskAttemptId attemptID) {
            return attempts.get(attemptID);
        }

        @Override
        public boolean isFinished() {
            switch(report.getTaskState()) {
                case SUCCEEDED:
                case KILLED:
                case FAILED:
                    return true;
            }
            return false;
        }

        @Override
        public boolean canCommit(TaskAttemptId taskAttemptID) {
            return false;
        }

        @Override
        public TaskState getState() {
            return report.getTaskState();
        }
    };
}

Example 20 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class MockJobs method newJob.

public static Job newJob(ApplicationId appID, int i, int n, int m, Path confFile, boolean hasFailedTasks) {
    final JobId id = newJobID(appID, i);
    final String name = newJobName();
    final JobReport report = newJobReport(id);
    final Map<TaskId, Task> tasks = newTasks(id, n, m, hasFailedTasks);
    final TaskCount taskCount = getTaskCount(tasks.values());
    final Counters counters = getCounters(tasks.values());
    final Path configFile = confFile;
    Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
    final Configuration conf = new Configuration();
    conf.set(JobACL.VIEW_JOB.getAclName(), "testuser");
    conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
    JobACLsManager aclsManager = new JobACLsManager(conf);
    tmpJobACLs = aclsManager.constructJobACLs(conf);
    final Map<JobACL, AccessControlList> jobACLs = tmpJobACLs;
    return new Job() {

        @Override
        public JobId getID() {
            return id;
        }

        @Override
        public String getName() {
            return name;
        }

        @Override
        public JobState getState() {
            return report.getJobState();
        }

        @Override
        public JobReport getReport() {
            return report;
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public Counters getAllCounters() {
            return counters;
        }

        @Override
        public Map<TaskId, Task> getTasks() {
            return tasks;
        }

        @Override
        public Task getTask(TaskId taskID) {
            return tasks.get(taskID);
        }

        @Override
        public int getTotalMaps() {
            return taskCount.maps;
        }

        @Override
        public int getTotalReduces() {
            return taskCount.reduces;
        }

        @Override
        public int getCompletedMaps() {
            return taskCount.completedMaps;
        }

        @Override
        public int getCompletedReduces() {
            return taskCount.completedReduces;
        }

        @Override
        public boolean isUber() {
            return false;
        }

        @Override
        public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(int fromEventId, int maxEvents) {
            return null;
        }

        @Override
        public TaskCompletionEvent[] getMapAttemptCompletionEvents(int startIndex, int maxEvents) {
            return null;
        }

        @Override
        public Map<TaskId, Task> getTasks(TaskType taskType) {
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public List<String> getDiagnostics() {
            return Collections.<String>emptyList();
        }

        @Override
        public boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) {
            return true;
        }

        @Override
        public String getUserName() {
            return "mock";
        }

        @Override
        public String getQueueName() {
            return "mockqueue";
        }

        @Override
        public Path getConfFile() {
            return configFile;
        }

        @Override
        public Map<JobACL, AccessControlList> getJobACLs() {
            return jobACLs;
        }

        @Override
        public List<AMInfo> getAMInfos() {
            List<AMInfo> amInfoList = new LinkedList<AMInfo>();
            amInfoList.add(createAMInfo(1));
            amInfoList.add(createAMInfo(2));
            return amInfoList;
        }

        @Override
        public Configuration loadConfFile() throws IOException {
            FileContext fc = FileContext.getFileContext(configFile.toUri(), conf);
            Configuration jobConf = new Configuration(false);
            jobConf.addResource(fc.open(configFile), configFile.toString());
            return jobConf;
        }

        @Override
        public void setQueueName(String queueName) {
        // do nothing
        }

        @Override
        public void setJobPriority(Priority priority) {
        // do nothing
        }
    };
}