Examples with Task org.apache.hadoop.mapreduce.v2.app.job.Task used on opensource projects

Example 91 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class MRApp method waitForInternalState.

public void waitForInternalState(TaskImpl task, TaskStateInternal finalState) throws Exception {
    int timeoutSecs = 0;
    TaskReport report = task.getReport();
    TaskStateInternal iState = task.getInternalState();
    while (!finalState.equals(iState) && timeoutSecs++ < 20) {
        System.out.println("Task Internal State is : " + iState + " Waiting for Internal state : " + finalState + "   progress : " + report.getProgress());
        Thread.sleep(500);
        report = task.getReport();
        iState = task.getInternalState();
    }
    System.out.println("Task Internal State is : " + iState);
    Assert.assertEquals("Task Internal state is not correct (timedout)", finalState, iState);
}

Example 92 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class MRApp method verifyCompleted.

public void verifyCompleted() {
    for (Job job : getContext().getAllJobs().values()) {
        JobReport jobReport = job.getReport();
        System.out.println("Job start time :" + jobReport.getStartTime());
        System.out.println("Job finish time :" + jobReport.getFinishTime());
        Assert.assertTrue("Job start time is not less than finish time", jobReport.getStartTime() <= jobReport.getFinishTime());
        Assert.assertTrue("Job finish time is in future", jobReport.getFinishTime() <= System.currentTimeMillis());
        for (Task task : job.getTasks().values()) {
            TaskReport taskReport = task.getReport();
            System.out.println("Task start time : " + taskReport.getStartTime());
            System.out.println("Task finish time : " + taskReport.getFinishTime());
            Assert.assertTrue("Task start time is not less than finish time", taskReport.getStartTime() <= taskReport.getFinishTime());
            for (TaskAttempt attempt : task.getAttempts().values()) {
                TaskAttemptReport attemptReport = attempt.getReport();
                Assert.assertTrue("Attempt start time is not less than finish time", attemptReport.getStartTime() <= attemptReport.getFinishTime());
            }
        }
    }
}

Example 93 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class TestMRApp method testCommitPending.

@Test
public void testCommitPending() throws Exception {
    MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true);
    Job job = app.submit(new Configuration());
    app.waitForState(job, JobState.RUNNING);
    Assert.assertEquals("Num tasks not correct", 1, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task task = it.next();
    app.waitForState(task, TaskState.RUNNING);
    TaskAttempt attempt = task.getAttempts().values().iterator().next();
    app.waitForState(attempt, TaskAttemptState.RUNNING);
    //send the commit pending signal to the task
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_COMMIT_PENDING));
    //wait for first attempt to commit pending
    app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING);
    //re-send the commit pending signal to the task
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_COMMIT_PENDING));
    //the task attempt should be still at COMMIT_PENDING
    app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING);
    //send the done signal to the task
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(task.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE));
    app.waitForState(job, JobState.SUCCEEDED);
}

Example 94 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class TestMRApp method testCompletedMapsForReduceSlowstart.

//@Test
public void testCompletedMapsForReduceSlowstart() throws Exception {
    MRApp app = new MRApp(2, 1, false, this.getClass().getName(), true);
    Configuration conf = new Configuration();
    //after half of the map completion, reduce will start
    conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.5f);
    //uberization forces full slowstart (1.0), so disable that
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    Job job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);
    //all maps would be running
    Assert.assertEquals("Num tasks not correct", 3, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task mapTask1 = it.next();
    Task mapTask2 = it.next();
    Task reduceTask = it.next();
    // all maps must be running
    app.waitForState(mapTask1, TaskState.RUNNING);
    app.waitForState(mapTask2, TaskState.RUNNING);
    TaskAttempt task1Attempt = mapTask1.getAttempts().values().iterator().next();
    TaskAttempt task2Attempt = mapTask2.getAttempts().values().iterator().next();
    //before sending the TA_DONE, event make sure attempt has come to 
    //RUNNING state
    app.waitForState(task1Attempt, TaskAttemptState.RUNNING);
    app.waitForState(task2Attempt, TaskAttemptState.RUNNING);
    // reduces must be in NEW state
    Assert.assertEquals("Reduce Task state not correct", TaskState.NEW, reduceTask.getReport().getTaskState());
    //send the done signal to the 1st map task
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(mapTask1.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE));
    //wait for first map task to complete
    app.waitForState(mapTask1, TaskState.SUCCEEDED);
    //Once the first map completes, it will schedule the reduces
    //now reduce must be running
    app.waitForState(reduceTask, TaskState.RUNNING);
    //send the done signal to 2nd map and the reduce to complete the job
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(mapTask2.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE));
    app.getContext().getEventHandler().handle(new TaskAttemptEvent(reduceTask.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE));
    app.waitForState(job, JobState.SUCCEEDED);
}

Example 95 with Task

use of org.apache.hadoop.mapreduce.v2.app.job.Task in project hadoop by apache.

the class MockJobs method getCounters.

public static Counters getCounters(Collection<Task> tasks) {
    List<Task> completedTasks = new ArrayList<Task>();
    for (Task task : tasks) {
        if (task.getCounters() != null) {
            completedTasks.add(task);
        }
    }
    Counters counters = new Counters();
    return JobImpl.incrTaskCounters(counters, completedTasks);
}