Examples with RMHeartbeatHandler org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler used on opensource projects

Example 1 with RMHeartbeatHandler

use of org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler in project hadoop by apache.

the class TestJobImpl method createCommitterEventHandler.

private static CommitterEventHandler createCommitterEventHandler(Dispatcher dispatcher, OutputCommitter committer) {
    final SystemClock clock = SystemClock.getInstance();
    AppContext appContext = mock(AppContext.class);
    when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {

        @Override
        public long getLastHeartbeatTime() {
            return clock.getTime();
        }

        @Override
        public void runOnNextHeartbeat(Runnable callback) {
            callback.run();
        }
    };
    ApplicationAttemptId id = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(id);
    CommitterEventHandler handler = new CommitterEventHandler(appContext, committer, heartbeatHandler);
    dispatcher.register(CommitterEventType.class, handler);
    return handler;
}

Example 2 with RMHeartbeatHandler

use of org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testCheckpointIDTracking.

@Test
public void testCheckpointIDTracking() throws IOException, InterruptedException {
    SystemClock clock = SystemClock.getInstance();
    org.apache.hadoop.mapreduce.v2.app.job.Task mockTask = mock(org.apache.hadoop.mapreduce.v2.app.job.Task.class);
    when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true);
    Job mockJob = mock(Job.class);
    when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    AppContext appCtx = mock(AppContext.class);
    when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
    when(appCtx.getClock()).thenReturn(clock);
    when(appCtx.getEventHandler()).thenReturn(ea);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy) {

        @Override
        protected void registerHeartbeatHandler(Configuration conf) {
            taskHeartbeatHandler = hbHandler;
        }
    };
    Configuration conf = new Configuration();
    conf.setBoolean(MRJobConfig.TASK_PREEMPTION, true);
    //conf.setBoolean("preemption.reduce", true);
    listener.init(conf);
    listener.start();
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
    List<Path> partialOut = new ArrayList<Path>();
    partialOut.add(new Path("/prev1"));
    partialOut.add(new Path("/prev2"));
    Counters counters = mock(Counters.class);
    final long CBYTES = 64L * 1024 * 1024;
    final long CTIME = 4344L;
    final Path CLOC = new Path("/test/1");
    Counter cbytes = mock(Counter.class);
    when(cbytes.getValue()).thenReturn(CBYTES);
    Counter ctime = mock(Counter.class);
    when(ctime.getValue()).thenReturn(CTIME);
    when(counters.findCounter(eq(EnumCounter.CHECKPOINT_BYTES))).thenReturn(cbytes);
    when(counters.findCounter(eq(EnumCounter.CHECKPOINT_MS))).thenReturn(ctime);
    // propagating a taskstatus that contains a checkpoint id
    TaskCheckpointID incid = new TaskCheckpointID(new FSCheckpointID(CLOC), partialOut, counters);
    listener.setCheckpointID(org.apache.hadoop.mapred.TaskID.downgrade(tid.getTaskID()), incid);
    // and try to get it back
    CheckpointID outcid = listener.getCheckpointID(tid.getTaskID());
    TaskCheckpointID tcid = (TaskCheckpointID) outcid;
    assertEquals(CBYTES, tcid.getCheckpointBytes());
    assertEquals(CTIME, tcid.getCheckpointTime());
    assertTrue(partialOut.containsAll(tcid.getPartialCommittedOutput()));
    assertTrue(tcid.getPartialCommittedOutput().containsAll(partialOut));
    //assert it worked
    assert outcid == incid;
    listener.stop();
}

Example 3 with RMHeartbeatHandler

use of org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler in project hadoop by apache.

the class TestTaskAttemptFinishingMonitor method testFinshingAttemptTimeout.

@Test
public void testFinshingAttemptTimeout() throws IOException, InterruptedException {
    SystemClock clock = SystemClock.getInstance();
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.TASK_EXIT_TIMEOUT, 100);
    conf.setInt(MRJobConfig.TASK_EXIT_TIMEOUT_CHECK_INTERVAL_MS, 10);
    AppContext appCtx = mock(AppContext.class);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    MockEventHandler eventHandler = new MockEventHandler();
    TaskAttemptFinishingMonitor taskAttemptFinishingMonitor = new TaskAttemptFinishingMonitor(eventHandler);
    taskAttemptFinishingMonitor.init(conf);
    taskAttemptFinishingMonitor.start();
    when(appCtx.getEventHandler()).thenReturn(eventHandler);
    when(appCtx.getNMHostname()).thenReturn("0.0.0.0");
    when(appCtx.getTaskAttemptFinishingMonitor()).thenReturn(taskAttemptFinishingMonitor);
    when(appCtx.getClock()).thenReturn(clock);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new TaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy);
    listener.init(conf);
    listener.start();
    JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
    TaskId tid = MRBuilderUtils.newTaskId(jid, 0, org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP);
    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
    appCtx.getTaskAttemptFinishingMonitor().register(attemptId);
    int check = 0;
    while (!eventHandler.timedOut && check++ < 10) {
        Thread.sleep(100);
    }
    taskAttemptFinishingMonitor.stop();
    assertTrue("Finishing attempt didn't time out.", eventHandler.timedOut);
}

Example 4 with RMHeartbeatHandler

use of org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testGetMapCompletionEvents.

@Test(timeout = 10000)
public void testGetMapCompletionEvents() throws IOException {
    TaskAttemptCompletionEvent[] empty = {};
    TaskAttemptCompletionEvent[] taskEvents = { createTce(0, true, TaskAttemptCompletionEventStatus.OBSOLETE), createTce(1, false, TaskAttemptCompletionEventStatus.FAILED), createTce(2, true, TaskAttemptCompletionEventStatus.SUCCEEDED), createTce(3, false, TaskAttemptCompletionEventStatus.FAILED) };
    TaskAttemptCompletionEvent[] mapEvents = { taskEvents[0], taskEvents[2] };
    Job mockJob = mock(Job.class);
    when(mockJob.getTaskAttemptCompletionEvents(0, 100)).thenReturn(taskEvents);
    when(mockJob.getTaskAttemptCompletionEvents(0, 2)).thenReturn(Arrays.copyOfRange(taskEvents, 0, 2));
    when(mockJob.getTaskAttemptCompletionEvents(2, 100)).thenReturn(Arrays.copyOfRange(taskEvents, 2, 4));
    when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(TypeConverter.fromYarn(mapEvents));
    when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(TypeConverter.fromYarn(mapEvents));
    when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(TypeConverter.fromYarn(empty));
    AppContext appCtx = mock(AppContext.class);
    when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy) {

        @Override
        protected void registerHeartbeatHandler(Configuration conf) {
            taskHeartbeatHandler = hbHandler;
        }
    };
    Configuration conf = new Configuration();
    listener.init(conf);
    listener.start();
    JobID jid = new JobID("12345", 1);
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
    MapTaskCompletionEventsUpdate update = listener.getMapCompletionEvents(jid, 0, 100, tid);
    assertEquals(2, update.events.length);
    update = listener.getMapCompletionEvents(jid, 0, 2, tid);
    assertEquals(2, update.events.length);
    update = listener.getMapCompletionEvents(jid, 2, 100, tid);
    assertEquals(0, update.events.length);
}

Example 5 with RMHeartbeatHandler

use of org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testCommitWindow.

@Test(timeout = 10000)
public void testCommitWindow() throws IOException {
    SystemClock clock = SystemClock.getInstance();
    org.apache.hadoop.mapreduce.v2.app.job.Task mockTask = mock(org.apache.hadoop.mapreduce.v2.app.job.Task.class);
    when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true);
    Job mockJob = mock(Job.class);
    when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask);
    AppContext appCtx = mock(AppContext.class);
    when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
    when(appCtx.getClock()).thenReturn(clock);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy) {

        @Override
        protected void registerHeartbeatHandler(Configuration conf) {
            taskHeartbeatHandler = hbHandler;
        }
    };
    Configuration conf = new Configuration();
    listener.init(conf);
    listener.start();
    // verify commit not allowed when RM heartbeat has not occurred recently
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
    boolean canCommit = listener.canCommit(tid);
    assertFalse(canCommit);
    verify(mockTask, never()).canCommit(any(TaskAttemptId.class));
    // verify commit allowed when RM heartbeat is recent
    when(rmHeartbeatHandler.getLastHeartbeatTime()).thenReturn(clock.getTime());
    canCommit = listener.canCommit(tid);
    assertTrue(canCommit);
    verify(mockTask, times(1)).canCommit(any(TaskAttemptId.class));
    listener.stop();
}