Examples with Task org.apache.hadoop.mapreduce.v2.app.job.Task used on opensource projects

Example 26 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class DefaultSpeculator method addSpeculativeAttempt.

//Add attempt to a given Task.
protected void addSpeculativeAttempt(TaskId taskID) {
    LOG.info("DefaultSpeculator.addSpeculativeAttempt -- we are speculating " + taskID);
    eventHandler.handle(new TaskEvent(taskID, TaskEventType.T_ADD_SPEC_ATTEMPT));
    mayHaveSpeculated.add(taskID);
}

Example 27 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class JobImpl method actOnUnusableNode.

private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
    // running reducers
    if (getInternalState() == JobStateInternal.RUNNING && !allReducersComplete()) {
        List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
        if (taskAttemptIdList != null) {
            String mesg = "TaskAttempt killed because it ran on unusable node " + nodeId;
            for (TaskAttemptId id : taskAttemptIdList) {
                if (TaskType.MAP == id.getTaskId().getTaskType()) {
                    // reschedule only map tasks because their outputs maybe unusable
                    LOG.info(mesg + ". AttemptId:" + id);
                    // Kill the attempt and indicate that next map attempt should be
                    // rescheduled (i.e. considered as a fast fail map).
                    eventHandler.handle(new TaskAttemptKillEvent(id, mesg, true));
                }
            }
        }
    }
// currently running task attempts on unusable nodes are handled in
// RMContainerAllocator
}

Example 28 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class CountersBlock method getCounters.

private void getCounters(AppContext ctx) {
    JobId jobID = null;
    TaskId taskID = null;
    String tid = $(TASK_ID);
    if (!tid.isEmpty()) {
        taskID = MRApps.toTaskID(tid);
        jobID = taskID.getJobId();
    } else {
        String jid = $(JOB_ID);
        if (jid != null && !jid.isEmpty()) {
            jobID = MRApps.toJobID(jid);
        }
    }
    if (jobID == null) {
        return;
    }
    job = ctx.getJob(jobID);
    if (job == null) {
        return;
    }
    if (taskID != null) {
        task = job.getTask(taskID);
        if (task == null) {
            return;
        }
        total = task.getCounters();
        return;
    }
    // Get all types of counters
    Map<TaskId, Task> tasks = job.getTasks();
    total = job.getAllCounters();
    boolean needTotalCounters = false;
    if (total == null) {
        total = new Counters();
        needTotalCounters = true;
    }
    map = new Counters();
    reduce = new Counters();
    for (Task t : tasks.values()) {
        Counters counters = t.getCounters();
        if (counters == null) {
            continue;
        }
        switch(t.getType()) {
            case MAP:
                map.incrAllCounters(counters);
                break;
            case REDUCE:
                reduce.incrAllCounters(counters);
                break;
        }
        if (needTotalCounters) {
            total.incrAllCounters(counters);
        }
    }
}

Example 29 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class JobBlock method render.

@Override
protected void render(Block html) {
    String jid = $(JOB_ID);
    if (jid.isEmpty()) {
        html.p()._("Sorry, can't do anything without a JobID.")._();
        return;
    }
    JobId jobID = MRApps.toJobID(jid);
    Job job = appContext.getJob(jobID);
    if (job == null) {
        html.p()._("Sorry, ", jid, " not found.")._();
        return;
    }
    List<AMInfo> amInfos = job.getAMInfos();
    String amString = amInfos.size() == 1 ? "ApplicationMaster" : "ApplicationMasters";
    JobInfo jinfo = new JobInfo(job, true);
    info("Job Overview")._("Job Name:", jinfo.getName())._("User Name:", jinfo.getUserName())._("Queue Name:", jinfo.getQueueName())._("State:", jinfo.getState())._("Uberized:", jinfo.isUberized())._("Started:", new Date(jinfo.getStartTime()))._("Elapsed:", StringUtils.formatTime(jinfo.getElapsedTime()));
    DIV<Hamlet> div = html._(InfoBlock.class).div(_INFO_WRAP);
    // MRAppMasters Table
    TABLE<DIV<Hamlet>> table = div.table("#job");
    table.tr().th(amString)._().tr().th(_TH, "Attempt Number").th(_TH, "Start Time").th(_TH, "Node").th(_TH, "Logs")._();
    for (AMInfo amInfo : amInfos) {
        AMAttemptInfo attempt = new AMAttemptInfo(amInfo, jinfo.getId(), jinfo.getUserName());
        table.tr().td(String.valueOf(attempt.getAttemptId())).td(new Date(attempt.getStartTime()).toString()).td().a(".nodelink", url(MRWebAppUtil.getYARNWebappScheme(), attempt.getNodeHttpAddress()), attempt.getNodeHttpAddress())._().td().a(".logslink", url(attempt.getLogsLink()), "logs")._()._();
    }
    table._();
    div._();
    html.div(_INFO_WRAP).table("#job").tr().th(_TH, "Task Type").th(_TH, "Progress").th(_TH, "Total").th(_TH, "Pending").th(_TH, "Running").th(_TH, "Complete")._().tr(_ODD).th("Map").td().div(_PROGRESSBAR).$title(// tooltip
    join(jinfo.getMapProgressPercent(), '%')).div(_PROGRESSBAR_VALUE).$style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._().td().a(url("tasks", jid, "m", "ALL"), String.valueOf(jinfo.getMapsTotal()))._().td().a(url("tasks", jid, "m", "PENDING"), String.valueOf(jinfo.getMapsPending()))._().td().a(url("tasks", jid, "m", "RUNNING"), String.valueOf(jinfo.getMapsRunning()))._().td().a(url("tasks", jid, "m", "COMPLETED"), String.valueOf(jinfo.getMapsCompleted()))._()._().tr(_EVEN).th("Reduce").td().div(_PROGRESSBAR).$title(// tooltip
    join(jinfo.getReduceProgressPercent(), '%')).div(_PROGRESSBAR_VALUE).$style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._().td().a(url("tasks", jid, "r", "ALL"), String.valueOf(jinfo.getReducesTotal()))._().td().a(url("tasks", jid, "r", "PENDING"), String.valueOf(jinfo.getReducesPending()))._().td().a(url("tasks", jid, "r", "RUNNING"), String.valueOf(jinfo.getReducesRunning()))._().td().a(url("tasks", jid, "r", "COMPLETED"), String.valueOf(jinfo.getReducesCompleted()))._()._()._().table("#job").tr().th(_TH, "Attempt Type").th(_TH, "New").th(_TH, "Running").th(_TH, "Failed").th(_TH, "Killed").th(_TH, "Successful")._().tr(_ODD).th("Maps").td().a(url("attempts", jid, "m", TaskAttemptStateUI.NEW.toString()), String.valueOf(jinfo.getNewMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.RUNNING.toString()), String.valueOf(jinfo.getRunningMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.FAILED.toString()), String.valueOf(jinfo.getFailedMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.KILLED.toString()), String.valueOf(jinfo.getKilledMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.SUCCESSFUL.toString()), String.valueOf(jinfo.getSuccessfulMapAttempts()))._()._().tr(_EVEN).th("Reduces").td().a(url("attempts", jid, "r", TaskAttemptStateUI.NEW.toString()), String.valueOf(jinfo.getNewReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.RUNNING.toString()), String.valueOf(jinfo.getRunningReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.FAILED.toString()), String.valueOf(jinfo.getFailedReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.KILLED.toString()), String.valueOf(jinfo.getKilledReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.SUCCESSFUL.toString()), String.valueOf(jinfo.getSuccessfulReduceAttempts()))._()._()._()._();
}

Example 30 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class SingleCounterBlock method populateMembers.

private void populateMembers(AppContext ctx) {
    JobId jobID = null;
    TaskId taskID = null;
    String tid = $(TASK_ID);
    if ($(TITLE).contains("MAPS")) {
        counterType = TaskType.MAP;
    } else if ($(TITLE).contains("REDUCES")) {
        counterType = TaskType.REDUCE;
    } else {
        counterType = null;
    }
    if (!tid.isEmpty()) {
        taskID = MRApps.toTaskID(tid);
        jobID = taskID.getJobId();
    } else {
        String jid = $(JOB_ID);
        if (!jid.isEmpty()) {
            jobID = MRApps.toJobID(jid);
        }
    }
    if (jobID == null) {
        return;
    }
    job = ctx.getJob(jobID);
    if (job == null) {
        return;
    }
    if (taskID != null) {
        task = job.getTask(taskID);
        if (task == null) {
            return;
        }
        for (Map.Entry<TaskAttemptId, TaskAttempt> entry : task.getAttempts().entrySet()) {
            long value = 0;
            Counters counters = entry.getValue().getCounters();
            CounterGroup group = (counters != null) ? counters.getGroup($(COUNTER_GROUP)) : null;
            if (group != null) {
                Counter c = group.findCounter($(COUNTER_NAME));
                if (c != null) {
                    value = c.getValue();
                }
            }
            values.put(MRApps.toString(entry.getKey()), value);
        }
        return;
    }
    // Get all types of counters
    Map<TaskId, Task> tasks = job.getTasks();
    for (Map.Entry<TaskId, Task> entry : tasks.entrySet()) {
        long value = 0;
        Counters counters = entry.getValue().getCounters();
        CounterGroup group = (counters != null) ? counters.getGroup($(COUNTER_GROUP)) : null;
        if (group != null) {
            Counter c = group.findCounter($(COUNTER_NAME));
            if (c != null) {
                value = c.getValue();
            }
        }
        if (counterType == null || counterType == entry.getValue().getType()) {
            values.put(MRApps.toString(entry.getKey()), value);
        }
    }
}