Examples with ControlledClock org.apache.hadoop.hive.llap.testhelpers.ControlledClock used on opensource projects

Example 1 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testDelayedLocalityNodeCommErrorDelayedAllocation.

@Test(timeout = 15000)
public void testDelayedLocalityNodeCommErrorDelayedAllocation() throws IOException, InterruptedException {
    Priority priority1 = Priority.newInstance(1);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(5000, hosts, 1, 1, 10000l, true);
    LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled delayedTaskSchedulerCallableControlled = (LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled) tsWrapper.ts.delayedTaskSchedulerCallable;
    ControlledClock clock = tsWrapper.getClock();
    clock.setTime(clock.getTime());
    // Fill up host1 with tasks. Leave host2 empty.
    try {
        tsWrapper.controlScheduler(true);
        TezTaskAttemptID task1 = tsWrapper.allocateTask(hostsH1, priority1);
        TezTaskAttemptID task2 = tsWrapper.allocateTask(hostsH1, priority1);
        // 1 more than capacity.
        TezTaskAttemptID task3 = tsWrapper.allocateTask(hostsH1, priority1);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        reset(tsWrapper.mockAppCallback);
        // Mark a task as failed due to a comm failure.
        tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.COMMUNICATION_ERROR);
        // Node1 has free capacity but is disabled. Node 2 has capcaity. Delay > re-enable tiemout
        tsWrapper.ensureNoChangeInTotalAllocations(2, 2000l);
    } finally {
        tsWrapper.shutdown();
    }
}

Example 2 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testTaskInfoDelay.

@Test(timeout = 10000)
public void testTaskInfoDelay() {
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf1 = new LlapTaskSchedulerService.LocalityDelayConf(3000);
    ControlledClock clock = new ControlledClock(new MonotonicClock());
    clock.setTime(clock.getTime());
    // With a timeout of 3000.
    LlapTaskSchedulerService.TaskInfo taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf1, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertFalse(taskInfo.shouldForceLocality());
    assertEquals(3000, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    clock.setTime(clock.getTime() + 500);
    assertEquals(2500, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    clock.setTime(clock.getTime() + 2500);
    assertEquals(0, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertFalse(taskInfo.shouldDelayForLocality(clock.getTime()));
    // No locality delay
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf2 = new LlapTaskSchedulerService.LocalityDelayConf(0);
    taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf2, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertFalse(taskInfo.shouldDelayForLocality(clock.getTime()));
    assertFalse(taskInfo.shouldForceLocality());
    assertTrue(taskInfo.getDelay(TimeUnit.MILLISECONDS) < 0);
    // Force locality
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf3 = new LlapTaskSchedulerService.LocalityDelayConf(-1);
    taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf3, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    assertTrue(taskInfo.shouldForceLocality());
    assertFalse(taskInfo.getDelay(TimeUnit.MILLISECONDS) < 0);
}

Example 3 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestTaskExecutorService method runPreemptionGraceTest.

private void runPreemptionGraceTest(MockRequest victim1, MockRequest victim2, int time) throws InterruptedException {
    MockRequest preemptor = createMockRequest(3, 1, 100, 100, true, 20000l, false);
    victim1.setSleepAfterKill();
    victim2.setSleepAfterKill();
    ControlledClock clock = new ControlledClock(new SystemClock());
    clock.setTime(0);
    TaskExecutorServiceForTest taskExecutorService = new TaskExecutorServiceForTest(2, 3, ShortestJobFirstComparator.class.getName(), true, mockMetrics, clock);
    taskExecutorService.init(new Configuration());
    taskExecutorService.start();
    try {
        taskExecutorService.schedule(victim1);
        awaitStartAndSchedulerRun(victim1, taskExecutorService);
        taskExecutorService.schedule(victim2);
        awaitStartAndSchedulerRun(victim2, taskExecutorService);
        taskExecutorService.schedule(preemptor);
        // Wait for scheduling to run a few times.
        taskExecutorService.waitForScheduleRuns(5);
        clock.setTime(time);
        // Wait for scheduling to run a few times.
        taskExecutorService.waitForScheduleRuns(5);
        victim1.unblockKill();
        victim2.unblockKill();
        preemptor.complete();
        preemptor.awaitEnd();
        TaskExecutorServiceForTest.InternalCompletionListenerForTest icl3 = taskExecutorService.getInternalCompletionListenerForTest(preemptor.getRequestId());
        icl3.awaitCompletion();
    } finally {
        taskExecutorService.shutDown(false);
    }
}

Example 4 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testDelayedLocalityDelayedAllocation.

@Test(timeout = 10000)
public void testDelayedLocalityDelayedAllocation() throws InterruptedException, IOException {
    Priority priority1 = Priority.newInstance(1);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, 10000l, true);
    LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled delayedTaskSchedulerCallableControlled = (LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled) tsWrapper.ts.delayedTaskSchedulerCallable;
    ControlledClock clock = tsWrapper.getClock();
    clock.setTime(clock.getTime());
    // Fill up host1 with tasks. Leave host2 empty.
    try {
        tsWrapper.controlScheduler(true);
        TezTaskAttemptID task1 = tsWrapper.allocateTask(hostsH1, priority1);
        TezTaskAttemptID task2 = tsWrapper.allocateTask(hostsH1, priority1);
        // 1 more than capacity.
        TezTaskAttemptID task3 = tsWrapper.allocateTask(hostsH1, priority1);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        reset(tsWrapper.mockAppCallback);
        // Move the clock forward 2000ms, and check the delayed queue
        // Past the timeout.
        clock.setTime(clock.getTime() + 2000l);
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_NOT_RUN, delayedTaskSchedulerCallableControlled.lastState);
        delayedTaskSchedulerCallableControlled.triggerGetNextTask();
        delayedTaskSchedulerCallableControlled.awaitGetNextTaskProcessing();
        // Verify that an attempt was made to schedule the task, but the decision was to skip scheduling
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_TIMEOUT_NOT_EXPIRED, delayedTaskSchedulerCallableControlled.lastState);
        assertFalse(delayedTaskSchedulerCallableControlled.shouldScheduleTaskTriggered);
        tsWrapper.deallocateTask(task1, true, null);
        // Node1 now has free capacity. task1 should be allocated to it.
        tsWrapper.awaitChangeInTotalAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        argumentCaptor = ArgumentCaptor.forClass(Object.class);
        ArgumentCaptor<Container> containerCaptor = ArgumentCaptor.forClass(Container.class);
        verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), containerCaptor.capture());
        assertEquals(1, argumentCaptor.getAllValues().size());
        assertEquals(task3, argumentCaptor.getAllValues().get(0));
        Container assignedContainer = containerCaptor.getValue();
        assertEquals(HOST1, assignedContainer.getNodeId().getHost());
        assertEquals(3, tsWrapper.ts.dagStats.getNumLocalAllocations());
        assertEquals(0, tsWrapper.ts.dagStats.getNumNonLocalAllocations());
        assertEquals(1, tsWrapper.ts.dagStats.getNumDelayedAllocations());
        assertEquals(3, tsWrapper.ts.dagStats.getNumAllocationsPerHost().get(HOST1).get());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 5 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testLocalityDelayTaskOrdering.

@Test(timeout = 10000)
public void testLocalityDelayTaskOrdering() throws InterruptedException, IOException {
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf = new LlapTaskSchedulerService.LocalityDelayConf(3000);
    ControlledClock clock = new ControlledClock(new MonotonicClock());
    clock.setTime(clock.getTime());
    DelayQueue<LlapTaskSchedulerService.TaskInfo> delayedQueue = new DelayQueue<>();
    LlapTaskSchedulerService.TaskInfo taskInfo1 = new LlapTaskSchedulerService.TaskInfo(localityDelayConf, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    clock.setTime(clock.getTime() + 1000);
    LlapTaskSchedulerService.TaskInfo taskInfo2 = new LlapTaskSchedulerService.TaskInfo(localityDelayConf, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    delayedQueue.add(taskInfo1);
    delayedQueue.add(taskInfo2);
    assertEquals(taskInfo1, delayedQueue.peek());
}