Examples with ControlledClock org.apache.hadoop.hive.llap.testhelpers.ControlledClock used on opensource projects

Example 6 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testDelayedQueeTaskSelectionAfterScheduled.

@Test(timeout = 10000)
public void testDelayedQueeTaskSelectionAfterScheduled() throws IOException, InterruptedException {
    Priority priority1 = Priority.newInstance(1);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, 10000l, true);
    LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled delayedTaskSchedulerCallableControlled = (LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled) tsWrapper.ts.delayedTaskSchedulerCallable;
    ControlledClock clock = tsWrapper.getClock();
    clock.setTime(clock.getTime());
    // Fill up host1 with tasks. Leave host2 empty.
    try {
        tsWrapper.controlScheduler(true);
        TezTaskAttemptID task1 = tsWrapper.allocateTask(hostsH1, priority1);
        TezTaskAttemptID task2 = tsWrapper.allocateTask(hostsH1, priority1);
        // 1 more than capacity.
        TezTaskAttemptID task3 = tsWrapper.allocateTask(hostsH1, priority1);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        // Simulate a 2s delay before finishing the task.
        clock.setTime(clock.getTime() + 2000);
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_NOT_RUN, delayedTaskSchedulerCallableControlled.lastState);
        delayedTaskSchedulerCallableControlled.triggerGetNextTask();
        delayedTaskSchedulerCallableControlled.awaitGetNextTaskProcessing();
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_TIMEOUT_NOT_EXPIRED, delayedTaskSchedulerCallableControlled.lastState);
        assertFalse(delayedTaskSchedulerCallableControlled.shouldScheduleTaskTriggered);
        reset(tsWrapper.mockAppCallback);
        // Now finish task1, which will make capacity for task3 to run. Nothing is coming out of the delayed queue yet.
        tsWrapper.deallocateTask(task1, true, null);
        tsWrapper.awaitLocalTaskAllocations(3);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        argumentCaptor = ArgumentCaptor.forClass(Object.class);
        ArgumentCaptor<Container> containerCaptor = ArgumentCaptor.forClass(Container.class);
        verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), containerCaptor.capture());
        assertEquals(1, argumentCaptor.getAllValues().size());
        assertEquals(task3, argumentCaptor.getAllValues().get(0));
        Container assignedContainer = containerCaptor.getValue();
        assertEquals(HOST1, assignedContainer.getNodeId().getHost());
        reset(tsWrapper.mockAppCallback);
        // Move the clock forward and trigger a run.
        // Set to start + 10000 which is the timeout
        clock.setTime(clock.getTime() + 8000);
        delayedTaskSchedulerCallableControlled.triggerGetNextTask();
        delayedTaskSchedulerCallableControlled.awaitGetNextTaskProcessing();
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_RETURNED_TASK, delayedTaskSchedulerCallableControlled.lastState);
        // Verify that an attempt was made to schedule the task, but the decision was to skip scheduling
        assertTrue(delayedTaskSchedulerCallableControlled.shouldScheduleTaskTriggered && !delayedTaskSchedulerCallableControlled.lastShouldScheduleTaskResult);
        // Ensure there's no more invocations.
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        verify(tsWrapper.mockAppCallback, never()).taskAllocated(any(Object.class), any(Object.class), any(Container.class));
    } finally {
        tsWrapper.shutdown();
    }
}

Example 7 with ControlledClock

use of org.apache.hadoop.hive.llap.testhelpers.ControlledClock in project hive by apache.

the class TestLlapTaskSchedulerService method testDelayedLocalityDelayedAllocation.

@Test(timeout = 10000)
public void testDelayedLocalityDelayedAllocation() throws InterruptedException, IOException {
    Priority priority1 = Priority.newInstance(1);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, 10000l, true);
    LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled delayedTaskSchedulerCallableControlled = (LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled) tsWrapper.ts.delayedTaskSchedulerCallable;
    ControlledClock clock = tsWrapper.getClock();
    clock.setTime(clock.getTime());
    // Fill up host1 with tasks. Leave host2 empty.
    try {
        tsWrapper.controlScheduler(true);
        TezTaskAttemptID task1 = tsWrapper.allocateTask(hostsH1, priority1);
        TezTaskAttemptID task2 = tsWrapper.allocateTask(hostsH1, priority1);
        // 1 more than capacity.
        TezTaskAttemptID task3 = tsWrapper.allocateTask(hostsH1, priority1);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        reset(tsWrapper.mockAppCallback);
        // Move the clock forward 2000ms, and check the delayed queue
        // Past the timeout.
        clock.setTime(clock.getTime() + 2000l);
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_NOT_RUN, delayedTaskSchedulerCallableControlled.lastState);
        delayedTaskSchedulerCallableControlled.triggerGetNextTask();
        delayedTaskSchedulerCallableControlled.awaitGetNextTaskProcessing();
        // Verify that an attempt was made to schedule the task, but the decision was to skip scheduling
        assertEquals(LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled.STATE_TIMEOUT_NOT_EXPIRED, delayedTaskSchedulerCallableControlled.lastState);
        assertFalse(delayedTaskSchedulerCallableControlled.shouldScheduleTaskTriggered);
        tsWrapper.deallocateTask(task1, true, null);
        // Node1 now has free capacity. task1 should be allocated to it.
        tsWrapper.awaitChangeInTotalAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        argumentCaptor = ArgumentCaptor.forClass(Object.class);
        ArgumentCaptor<Container> containerCaptor = ArgumentCaptor.forClass(Container.class);
        verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), containerCaptor.capture());
        assertEquals(1, argumentCaptor.getAllValues().size());
        assertEquals(task3, argumentCaptor.getAllValues().get(0));
        Container assignedContainer = containerCaptor.getValue();
        assertEquals(HOST1, assignedContainer.getNodeId().getHost());
        assertEquals(3, tsWrapper.ts.dagStats.numLocalAllocations);
        assertEquals(0, tsWrapper.ts.dagStats.numNonLocalAllocations);
        assertEquals(1, tsWrapper.ts.dagStats.numDelayedAllocations);
        assertEquals(3, tsWrapper.ts.dagStats.numAllocationsPerHost.get(HOST1).get());
    } finally {
        tsWrapper.shutdown();
    }
}