Examples with TaskInfo org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo used on opensource projects

Example 6 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testGuaranteedTransfer.

@Test(timeout = 10000)
public void testGuaranteedTransfer() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // low pri task. When the high pri finishes, low pri gets the duck back.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1);
        assertTrue(ti1.isGuaranteed());
        assertEquals(State.ASSIGNED, ti1.getState());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.allocateTask(task2, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertTrue(ti2.isGuaranteed());
        assertFalse(ti1.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertTrue(ti1.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 7 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testUpdateOnFinishingTask.

@Test(timeout = 10000)
public void testUpdateOnFinishingTask() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // The update fails because the task has terminated on the node.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2);
        // Concurrent increase and termination, increase fails.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        // Not updated yet.
        assertFalse(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.ts.handleUpdateResult(ti1, false);
        // We must have the duck still; it should just go to the other task.
        assertTrue(ti2.isGuaranteed());
        assertTrue(ti2.isUpdateInProgress());
        tsWrapper.ts.handleUpdateResult(ti2, false);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        // Same; with the termination after the failed update, we should maintain the correct count.
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 8 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testConcurrentUpdates.

@Test(timeout = 20000)
public void testConcurrentUpdates() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // the message callback should undo the change.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed());
        // Boring scenario #1 - two concurrent increases.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        // Not updated yet.
        assertFalse(ti1.getLastSetGuaranteed());
        assertFalse(ti2.isGuaranteed());
        // We are now "sending" a message... update again, "return" both callbacks.
        tsWrapper.ts.updateGuaranteedCount(2);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti2.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.handleUpdateResult(ti2, true);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti2.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        assertTrue(ti2.getLastSetGuaranteed());
        // Boring scenario #2 - two concurrent revokes. Same as above.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        assertFalse(ti2.isGuaranteed());
        // Not updated yet.
        assertTrue(ti2.getLastSetGuaranteed());
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.ts.waitForMessagesSent(1);
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti2.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.handleUpdateResult(ti2, true);
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti2.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertFalse(ti2.getLastSetGuaranteed());
        // Concurrent increase and revocation, then another increase - after the message is sent.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        // Not updated yet.
        assertFalse(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        tsWrapper.ts.updateGuaranteedCount(0);
        // We are revoking from an updating task.
        tsWrapper.ts.assertNoMessagesSent();
        assertFalse(ti1.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        // We should send a message to undo what we just did.
        tsWrapper.ts.waitForMessagesSent(1);
        assertFalse(ti1.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.assertNoMessagesSent();
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.assertNoMessagesSent();
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 9 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testUpdateWithError.

@Test(timeout = 10000)
public void testUpdateWithError() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // The update has failed; we'd try with another candidate first, but only at the same priority.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task3 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed());
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        TaskInfo tiHigher = ti1.isGuaranteed() ? ti1 : ti2, tiLower = (tiHigher == ti1) ? ti2 : ti1;
        assertTrue(tiHigher.isGuaranteed());
        // Not updated yet.
        assertFalse(tiHigher.getLastSetGuaranteed());
        assertTrue(tiHigher.isUpdateInProgress());
        // Update has failed. We should try task2.
        tsWrapper.ts.handleUpdateResult(tiHigher, false);
        tsWrapper.ts.waitForMessagesSent(1);
        assertFalse(tiHigher.isGuaranteed());
        assertFalse(tiHigher.getLastSetGuaranteed());
        assertFalse(tiHigher.isUpdateInProgress());
        assertTrue(tiLower.isGuaranteed());
        assertFalse(tiLower.getLastSetGuaranteed());
        assertTrue(tiLower.isUpdateInProgress());
        // Fail the 2nd update too to get rid of the duck for the next test.
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.ts.handleUpdateResult(tiLower, false);
        tsWrapper.ts.assertNoMessagesSent();
        // Now run a lower priority task.
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.allocateTask(task3, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(3);
        TaskInfo ti3 = tsWrapper.ts.getTaskInfo(task3);
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        // Update has failed. We won't try a low pri task.
        tsWrapper.ts.handleUpdateResult(ti1, false);
        assertTrue(ti1.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertFalse(ti3.isGuaranteed());
        assertFalse(ti3.isUpdateInProgress());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 10 with TaskInfo

use of org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.TaskInfo in project hive by apache.

the class TestLlapTaskSchedulerService method testConcurrentUpdateWithError.

@Test(timeout = 10000)
public void testConcurrentUpdateWithError() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // The update has failed but the state has changed since then - no retry needed.
    try {
        Priority highPri = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1);
        assertFalse(ti1.isGuaranteed());
        // Concurrent increase and revocation, increase fails - no revocation is needed.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.isGuaranteed());
        // Not updated yet.
        assertFalse(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        tsWrapper.ts.updateGuaranteedCount(0);
        // We are revoking from an updating task.
        tsWrapper.ts.assertNoMessagesSent();
        assertFalse(ti1.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, false);
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        tsWrapper.ts.assertNoMessagesSent();
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}