Examples with TaskId org.apache.hadoop.mapreduce.v2.api.records.TaskId used on opensource projects

Example 91 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestFail method testTimedOutTask.

@Test
public //All Task attempts are timed out, leading to Job failure
void testTimedOutTask() throws Exception {
    MRApp app = new TimeOutTaskMRApp(1, 0);
    Configuration conf = new Configuration();
    int maxAttempts = 2;
    conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
    // disable uberization (requires entire job to be reattempted, so max for
    // subtask attempts is overridden to 1)
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    Job job = app.submit(conf);
    app.waitForState(job, JobState.FAILED);
    Map<TaskId, Task> tasks = job.getTasks();
    Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
    Task task = tasks.values().iterator().next();
    Assert.assertEquals("Task state not correct", TaskState.FAILED, task.getReport().getTaskState());
    Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts();
    Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts.size());
    for (TaskAttempt attempt : attempts.values()) {
        Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED, attempt.getReport().getTaskAttemptState());
    }
}

Example 92 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class NavBlock method render.

@Override
protected void render(Block html) {
    String rmweb = $(RM_WEB);
    DIV<Hamlet> nav = html.div("#nav").h3("Cluster").ul().li().a(url(rmweb, "cluster", "cluster"), "About")._().li().a(url(rmweb, "cluster", "apps"), "Applications")._().li().a(url(rmweb, "cluster", "scheduler"), "Scheduler")._()._().h3("Application").ul().li().a(url("app/info"), "About")._().li().a(url("app"), "Jobs")._()._();
    if (app.getJob() != null) {
        String jobid = MRApps.toString(app.getJob().getID());
        List<AMInfo> amInfos = app.getJob().getAMInfos();
        AMInfo thisAmInfo = amInfos.get(amInfos.size() - 1);
        String nodeHttpAddress = thisAmInfo.getNodeManagerHost() + ":" + thisAmInfo.getNodeManagerHttpPort();
        nav.h3("Job").ul().li().a(url("job", jobid), "Overview")._().li().a(url("jobcounters", jobid), "Counters")._().li().a(url("conf", jobid), "Configuration")._().li().a(url("tasks", jobid, "m"), "Map tasks")._().li().a(url("tasks", jobid, "r"), "Reduce tasks")._().li().a(".logslink", url(MRWebAppUtil.getYARNWebappScheme(), nodeHttpAddress, "node", "containerlogs", thisAmInfo.getContainerId().toString(), app.getJob().getUserName()), "AM Logs")._()._();
        if (app.getTask() != null) {
            String taskid = MRApps.toString(app.getTask().getID());
            nav.h3("Task").ul().li().a(url("task", taskid), "Task Overview")._().li().a(url("taskcounters", taskid), "Counters")._()._();
        }
    }
    nav.h3("Tools").ul().li().a("/conf", "Configuration")._().li().a("/logs", "Local logs")._().li().a("/stacks", "Server stacks")._().li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
}

Example 93 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class JobCounterInfo method getCounters.

private void getCounters(AppContext ctx, Job job) {
    if (job == null) {
        return;
    }
    total = job.getAllCounters();
    boolean needTotalCounters = false;
    if (total == null) {
        total = new Counters();
        needTotalCounters = true;
    }
    map = new Counters();
    reduce = new Counters();
    // Get all types of counters
    Map<TaskId, Task> tasks = job.getTasks();
    for (Task t : tasks.values()) {
        Counters counters = t.getCounters();
        if (counters == null) {
            continue;
        }
        switch(t.getType()) {
            case MAP:
                map.incrAllCounters(counters);
                break;
            case REDUCE:
                reduce.incrAllCounters(counters);
                break;
        }
        if (needTotalCounters) {
            total.incrAllCounters(counters);
        }
    }
}

Example 94 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class JobInfo method countTasksAndAttempts.

/**
   * Go through a job and update the member variables with counts for
   * information to output in the page.
   *
   * @param job
   *          the job to get counts for.
   */
private void countTasksAndAttempts(Job job) {
    final Map<TaskId, Task> tasks = job.getTasks();
    if (tasks == null) {
        return;
    }
    for (Task task : tasks.values()) {
        switch(task.getType()) {
            case MAP:
                // Task counts
                switch(task.getState()) {
                    case RUNNING:
                        ++this.mapsRunning;
                        break;
                    case SCHEDULED:
                        ++this.mapsPending;
                        break;
                    default:
                        break;
                }
                break;
            case REDUCE:
                // Task counts
                switch(task.getState()) {
                    case RUNNING:
                        ++this.reducesRunning;
                        break;
                    case SCHEDULED:
                        ++this.reducesPending;
                        break;
                    default:
                        break;
                }
                break;
            default:
                throw new IllegalStateException("Task type is neither map nor reduce: " + task.getType());
        }
        // Attempts counts
        Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
        int newAttempts, running, successful, failed, killed;
        for (TaskAttempt attempt : attempts.values()) {
            newAttempts = 0;
            running = 0;
            successful = 0;
            failed = 0;
            killed = 0;
            if (TaskAttemptStateUI.NEW.correspondsTo(attempt.getState())) {
                ++newAttempts;
            } else if (TaskAttemptStateUI.RUNNING.correspondsTo(attempt.getState())) {
                ++running;
            } else if (TaskAttemptStateUI.SUCCESSFUL.correspondsTo(attempt.getState())) {
                ++successful;
            } else if (TaskAttemptStateUI.FAILED.correspondsTo(attempt.getState())) {
                ++failed;
            } else if (TaskAttemptStateUI.KILLED.correspondsTo(attempt.getState())) {
                ++killed;
            }
            switch(task.getType()) {
                case MAP:
                    this.newMapAttempts += newAttempts;
                    this.runningMapAttempts += running;
                    this.successfulMapAttempts += successful;
                    this.failedMapAttempts += failed;
                    this.killedMapAttempts += killed;
                    break;
                case REDUCE:
                    this.newReduceAttempts += newAttempts;
                    this.runningReduceAttempts += running;
                    this.successfulReduceAttempts += successful;
                    this.failedReduceAttempts += failed;
                    this.killedReduceAttempts += killed;
                    break;
                default:
                    throw new IllegalStateException("Task type neither map nor reduce: " + task.getType());
            }
        }
    }
}

Example 95 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class CheckpointAMPreemptionPolicy method updateCheckpointCounters.

@SuppressWarnings({ "unchecked" })
private void updateCheckpointCounters(TaskId taskId, TaskCheckpointID cid) {
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINTS, 1);
    eventHandler.handle(jce);
    jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINT_BYTES, cid.getCheckpointBytes());
    eventHandler.handle(jce);
    jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINT_TIME, cid.getCheckpointTime());
    eventHandler.handle(jce);
}