Examples with SystemClock org.apache.hadoop.yarn.util.SystemClock used on opensource projects

Example 6 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestQueueManager method setUp.

@Before
public void setUp() throws Exception {
    conf = new FairSchedulerConfiguration();
    scheduler = mock(FairScheduler.class);
    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
    // Set up some queues to test default child max resource inheritance
    allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test");
    allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA");
    allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB");
    when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
    when(scheduler.getConf()).thenReturn(conf);
    SystemClock clock = SystemClock.getInstance();
    when(scheduler.getClock()).thenReturn(clock);
    notEmptyQueues = new HashSet<>();
    queueManager = new QueueManager(scheduler) {

        @Override
        public boolean isEmpty(FSQueue queue) {
            return !notEmptyQueues.contains(queue);
        }
    };
    FSQueueMetrics.forQueue("root", null, true, conf);
    queueManager.initialize(conf);
}

Example 7 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hive by apache.

the class TestTaskExecutorService method runPreemptionGraceTest.

private void runPreemptionGraceTest(MockRequest victim1, MockRequest victim2, int time) throws InterruptedException {
    MockRequest preemptor = createMockRequest(3, 1, 100, 100, true, 20000l);
    victim1.setSleepAfterKill();
    victim2.setSleepAfterKill();
    ControlledClock clock = new ControlledClock(new SystemClock());
    clock.setTime(0);
    TaskExecutorServiceForTest taskExecutorService = new TaskExecutorServiceForTest(2, 3, ShortestJobFirstComparator.class.getName(), true, clock);
    taskExecutorService.init(new Configuration());
    taskExecutorService.start();
    try {
        taskExecutorService.schedule(victim1);
        awaitStartAndSchedulerRun(victim1, taskExecutorService);
        taskExecutorService.schedule(victim2);
        awaitStartAndSchedulerRun(victim2, taskExecutorService);
        taskExecutorService.schedule(preemptor);
        // Wait for scheduling to run a few times.
        taskExecutorService.waitForScheduleRuns(5);
        clock.setTime(time);
        // Wait for scheduling to run a few times.
        taskExecutorService.waitForScheduleRuns(5);
        victim1.unblockKill();
        victim2.unblockKill();
        preemptor.complete();
        preemptor.awaitEnd();
        TaskExecutorServiceForTest.InternalCompletionListenerForTest icl3 = taskExecutorService.getInternalCompletionListenerForTest(preemptor.getRequestId());
        icl3.awaitCompletion();
    } finally {
        taskExecutorService.shutDown(false);
    }
}

Example 8 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestTaskAttemptFinishingMonitor method testFinshingAttemptTimeout.

@Test
public void testFinshingAttemptTimeout() throws IOException, InterruptedException {
    SystemClock clock = SystemClock.getInstance();
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.TASK_EXIT_TIMEOUT, 100);
    conf.setInt(MRJobConfig.TASK_EXIT_TIMEOUT_CHECK_INTERVAL_MS, 10);
    AppContext appCtx = mock(AppContext.class);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    MockEventHandler eventHandler = new MockEventHandler();
    TaskAttemptFinishingMonitor taskAttemptFinishingMonitor = new TaskAttemptFinishingMonitor(eventHandler);
    taskAttemptFinishingMonitor.init(conf);
    taskAttemptFinishingMonitor.start();
    when(appCtx.getEventHandler()).thenReturn(eventHandler);
    when(appCtx.getNMHostname()).thenReturn("0.0.0.0");
    when(appCtx.getTaskAttemptFinishingMonitor()).thenReturn(taskAttemptFinishingMonitor);
    when(appCtx.getClock()).thenReturn(clock);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new TaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy);
    listener.init(conf);
    listener.start();
    JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
    TaskId tid = MRBuilderUtils.newTaskId(jid, 0, org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP);
    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
    appCtx.getTaskAttemptFinishingMonitor().register(attemptId);
    int check = 0;
    while (!eventHandler.timedOut && check++ < 10) {
        Thread.sleep(100);
    }
    taskAttemptFinishingMonitor.stop();
    assertTrue("Finishing attempt didn't time out.", eventHandler.timedOut);
}

Example 9 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testCommitWindow.

@Test(timeout = 10000)
public void testCommitWindow() throws IOException {
    SystemClock clock = SystemClock.getInstance();
    org.apache.hadoop.mapreduce.v2.app.job.Task mockTask = mock(org.apache.hadoop.mapreduce.v2.app.job.Task.class);
    when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true);
    Job mockJob = mock(Job.class);
    when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask);
    AppContext appCtx = mock(AppContext.class);
    when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
    when(appCtx.getClock()).thenReturn(clock);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy) {

        @Override
        protected void registerHeartbeatHandler(Configuration conf) {
            taskHeartbeatHandler = hbHandler;
        }
    };
    Configuration conf = new Configuration();
    listener.init(conf);
    listener.start();
    // verify commit not allowed when RM heartbeat has not occurred recently
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
    boolean canCommit = listener.canCommit(tid);
    assertFalse(canCommit);
    verify(mockTask, never()).canCommit(any(TaskAttemptId.class));
    // verify commit allowed when RM heartbeat is recent
    when(rmHeartbeatHandler.getLastHeartbeatTime()).thenReturn(clock.getTime());
    canCommit = listener.canCommit(tid);
    assertTrue(canCommit);
    verify(mockTask, times(1)).canCommit(any(TaskAttemptId.class));
    listener.stop();
}

Example 10 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestCommitterEventHandler method testCommitWindow.

@Test
public void testCommitWindow() throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    TestingJobEventHandler jeh = new TestingJobEventHandler();
    dispatcher.register(JobEventType.class, jeh);
    SystemClock clock = SystemClock.getInstance();
    AppContext appContext = mock(AppContext.class);
    ApplicationAttemptId attemptid = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
    when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    OutputCommitter committer = mock(OutputCommitter.class);
    TestingRMHeartbeatHandler rmhh = new TestingRMHeartbeatHandler();
    CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh);
    ceh.init(conf);
    ceh.start();
    // verify trying to commit when RM heartbeats are stale does not commit
    ceh.handle(new CommitterJobCommitEvent(null, null));
    long timeToWaitMs = 5000;
    while (rmhh.getNumCallbacks() != 1 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not register a heartbeat callback", 1, rmhh.getNumCallbacks());
    verify(committer, never()).commitJob(any(JobContext.class));
    Assert.assertEquals("committer should not have committed", 0, jeh.numCommitCompletedEvents);
    // set a fresh heartbeat and verify commit completes
    rmhh.setLastHeartbeatTime(clock.getTime());
    timeToWaitMs = 5000;
    while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not complete commit after RM hearbeat", 1, jeh.numCommitCompletedEvents);
    verify(committer, times(1)).commitJob(any(JobContext.class));
    //Clean up so we can try to commit again (Don't do this at home)
    cleanup();
    // try to commit again and verify it goes through since the heartbeat
    // is still fresh
    ceh.handle(new CommitterJobCommitEvent(null, null));
    timeToWaitMs = 5000;
    while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not commit", 2, jeh.numCommitCompletedEvents);
    verify(committer, times(2)).commitJob(any(JobContext.class));
    ceh.stop();
    dispatcher.stop();
}