Examples with TaskId org.apache.hadoop.mapreduce.v2.api.records.TaskId used on opensource projects

Example 46 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestTaskImpl method scheduleTaskAttempt.

private void scheduleTaskAttempt(TaskId taskId) {
    mockTask.handle(new TaskEvent(taskId, TaskEventType.T_SCHEDULE));
    assertTaskScheduledState();
    assertTaskAttemptAvataar(Avataar.VIRGIN);
}

Example 47 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestTaskImpl method testTaskProgress.

@Test
public void testTaskProgress() {
    LOG.info("--- START: testTaskProgress ---");
    mockTask = createMockTask(TaskType.MAP);
    // launch task
    TaskId taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    float progress = 0f;
    assert (mockTask.getProgress() == progress);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    // update attempt1 
    progress = 50f;
    updateLastAttemptProgress(progress);
    assert (mockTask.getProgress() == progress);
    progress = 100f;
    updateLastAttemptProgress(progress);
    assert (mockTask.getProgress() == progress);
    progress = 0f;
    // mark first attempt as killed
    updateLastAttemptState(TaskAttemptState.KILLED);
    assert (mockTask.getProgress() == progress);
    // kill first attempt 
    // should trigger a new attempt
    // as no successful attempts 
    killRunningTaskAttempt(getLastAttempt().getAttemptId());
    assert (taskAttempts.size() == 2);
    assert (mockTask.getProgress() == 0f);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    progress = 50f;
    updateLastAttemptProgress(progress);
    assert (mockTask.getProgress() == progress);
}

Example 48 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestTaskImpl method testFailureDuringTaskAttemptCommit.

@Test
public void testFailureDuringTaskAttemptCommit() {
    mockTask = createMockTask(TaskType.MAP);
    TaskId taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    updateLastAttemptState(TaskAttemptState.COMMIT_PENDING);
    commitTaskAttempt(getLastAttempt().getAttemptId());
    // During the task attempt commit there is an exception which causes
    // the attempt to fail
    updateLastAttemptState(TaskAttemptState.FAILED);
    failRunningTaskAttempt(getLastAttempt().getAttemptId());
    assertEquals(2, taskAttempts.size());
    updateLastAttemptState(TaskAttemptState.SUCCEEDED);
    commitTaskAttempt(getLastAttempt().getAttemptId());
    mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
    assertFalse("First attempt should not commit", mockTask.canCommit(taskAttempts.get(0).getAttemptId()));
    assertTrue("Second attempt should commit", mockTask.canCommit(getLastAttempt().getAttemptId()));
    assertTaskSucceededState();
}

Example 49 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestTaskImpl method testKillSuccessfulTask.

@Test
public void testKillSuccessfulTask() {
    LOG.info("--- START: testKillSuccesfulTask ---");
    mockTask = createMockTask(TaskType.MAP);
    TaskId taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    commitTaskAttempt(getLastAttempt().getAttemptId());
    mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
    assertTaskSucceededState();
    mockTask.handle(new TaskEvent(taskId, TaskEventType.T_KILL));
    assertTaskSucceededState();
}

Example 50 with TaskId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskId in project hadoop by apache.

the class TestContainerLauncher method testPoolSize.

@Test(timeout = 10000)
public void testPoolSize() throws InterruptedException {
    ApplicationId appId = ApplicationId.newInstance(12345, 67);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 3);
    JobId jobId = MRBuilderUtils.newJobId(appId, 8);
    TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
    AppContext context = mock(AppContext.class);
    CustomContainerLauncher containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(new Configuration());
    containerLauncher.start();
    ThreadPoolExecutor threadPool = containerLauncher.getThreadPool();
    // No events yet
    Assert.assertEquals(containerLauncher.initialPoolSize, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE);
    Assert.assertEquals(0, threadPool.getPoolSize());
    Assert.assertEquals(containerLauncher.initialPoolSize, threadPool.getCorePoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 10);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Same set of hosts, so no change
    containerLauncher.finishEventHandling = true;
    int timeOut = 0;
    while (containerLauncher.numEventsProcessed.get() < 10 && timeOut++ < 200) {
        LOG.info("Waiting for number of events processed to become " + 10 + ". It is now " + containerLauncher.numEventsProcessed.get() + ". Timeout is " + timeOut);
        Thread.sleep(1000);
    }
    Assert.assertEquals(10, containerLauncher.numEventsProcessed.get());
    containerLauncher.finishEventHandling = false;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i + 10);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i + 10);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 20);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Different hosts, there should be an increase in core-thread-pool size to
    // 21(11hosts+10buffer)
    // Core pool size should be 21 but the live pool size should be only 11.
    containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
    containerLauncher.finishEventHandling = false;
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 21);
    TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 21);
    containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host11:1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    waitForEvents(containerLauncher, 21);
    Assert.assertEquals(11, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.stop();
    // change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
    // and verify initialPoolSize value.
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE, 20);
    containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(conf);
    Assert.assertEquals(containerLauncher.initialPoolSize, 20);
}