Examples with TaskInfo org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo used on opensource projects

Example 1 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testGuaranteedScheduling.

@Test(timeout = 10000)
public void testGuaranteedScheduling() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // When the first one finishes, the duck goes to the 2nd, and then becomes unused.
    try {
        Priority priority = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = new Object(), clientCookie2 = new Object();
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, priority, clientCookie1);
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti = tsWrapper.ts.getTaskInfo(task1);
        assertTrue(ti.isGuaranteed());
        assertEquals(State.ASSIGNED, ti.getState());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.allocateTask(task2, null, priority, clientCookie2);
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti2.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertTrue(ti2.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 2 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testConcurrentUpdatesBeforeMessage.

@Test(timeout = 10000)
public void testConcurrentUpdatesBeforeMessage() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // before the message is sent; no message should ever be sent.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed());
        // Concurrent increase and revocation - before the message is sent.
        tsWrapper.ts.clearTestCounts();
        tsWrapper.ts.setDelayCheckAndSend(true);
        Thread updateThread = new Thread(new Runnable() {

            @Override
            public void run() {
                tsWrapper.ts.updateGuaranteedCount(1);
            }
        }, "test-update-thread");
        // This should eventually hang in the delay code.
        updateThread.start();
        // From the background thread.
        tsWrapper.ts.waitForCheckAndSendCall(1);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertFalse(ti1.getLastSetGuaranteed());
        // This won't go into checkAndSend.
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.ts.assertNoMessagesSent();
        // Release the background thread.
        tsWrapper.ts.setDelayCheckAndSend(false);
        updateThread.join();
        // No message is needed.
        tsWrapper.ts.assertNoMessagesSent();
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        // Concurrent revocation and increase - before the message is sent.
        // First, actually give it a duck.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.clearTestCounts();
        assertTrue(ti1.isGuaranteed() && ti1.getLastSetGuaranteed());
        tsWrapper.ts.setDelayCheckAndSend(true);
        updateThread = new Thread(new Runnable() {

            @Override
            public void run() {
                tsWrapper.ts.updateGuaranteedCount(0);
            }
        }, "test-update-thread");
        // This should eventually hang in the delay code.
        updateThread.start();
        tsWrapper.ts.waitForCheckAndSendCall(1);
        assertFalse(ti1.isGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertTrue(ti1.getLastSetGuaranteed());
        // This won't go into checkAndSend.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.assertNoMessagesSent();
        // Release the background thread.
        tsWrapper.ts.setDelayCheckAndSend(false);
        updateThread.join();
        // No message is needed.
        tsWrapper.ts.assertNoMessagesSent();
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 3 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testHeartbeatInconsistency.

@Test(timeout = 10000)
public void testHeartbeatInconsistency() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // Guaranteed flag is inconsistent based on heartbeat - another message should be send.
    try {
        Priority highPri = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1);
        assertFalse(ti1.isGuaranteed());
        // Heartbeat indicates task has a duck - this must be reverted.
        tsWrapper.ts.taskInfoUpdated(task1, true);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertFalse(ti1.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        assertFalse(ti1.getLastSetGuaranteed());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 4 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testChangeGuaranteedTotal.

@Test(timeout = 10000)
public void testChangeGuaranteedTotal() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // Then revoke similarly in steps (1, 4, 1), with the opposite effect.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task3 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.allocateTask(task3, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(3);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2), ti3 = tsWrapper.ts.getTaskInfo(task3);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed() || ti3.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(2);
        assertTrue(ti1.isGuaranteed());
        // This particular test doesn't care which of the lower pri tasks gets the duck.
        TaskInfo ti23High = ti2.isGuaranteed() ? ti2 : ti3, ti23Low = (ti2 == ti23High) ? ti3 : ti2;
        assertTrue(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(4);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(6);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(3, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(5);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(2, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(1);
        assertTrue(ti1.isGuaranteed());
        assertFalse(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(0);
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task3, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 5 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testTaskInfoDelay.

@Test(timeout = 10000)
public void testTaskInfoDelay() {
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf1 = new LlapTaskSchedulerService.LocalityDelayConf(3000);
    ControlledClock clock = new ControlledClock(new MonotonicClock());
    clock.setTime(clock.getTime());
    // With a timeout of 3000.
    LlapTaskSchedulerService.TaskInfo taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf1, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertFalse(taskInfo.shouldForceLocality());
    assertEquals(3000, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    clock.setTime(clock.getTime() + 500);
    assertEquals(2500, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    clock.setTime(clock.getTime() + 2500);
    assertEquals(0, taskInfo.getDelay(TimeUnit.MILLISECONDS));
    assertFalse(taskInfo.shouldDelayForLocality(clock.getTime()));
    // No locality delay
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf2 = new LlapTaskSchedulerService.LocalityDelayConf(0);
    taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf2, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertFalse(taskInfo.shouldDelayForLocality(clock.getTime()));
    assertFalse(taskInfo.shouldForceLocality());
    assertTrue(taskInfo.getDelay(TimeUnit.MILLISECONDS) < 0);
    // Force locality
    LlapTaskSchedulerService.LocalityDelayConf localityDelayConf3 = new LlapTaskSchedulerService.LocalityDelayConf(-1);
    taskInfo = new LlapTaskSchedulerService.TaskInfo(localityDelayConf3, clock, new Object(), new Object(), mock(Priority.class), mock(Resource.class), null, null, clock.getTime(), null);
    assertTrue(taskInfo.shouldDelayForLocality(clock.getTime()));
    assertTrue(taskInfo.shouldForceLocality());
    assertFalse(taskInfo.getDelay(TimeUnit.MILLISECONDS) < 0);
}