Examples with TezTaskAttemptID org.apache.tez.dag.records.TezTaskAttemptID used on opensource projects

Example 76 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testConcurrentUpdatesBeforeMessage.

@Test(timeout = 10000)
public void testConcurrentUpdatesBeforeMessage() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // before the message is sent; no message should ever be sent.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed());
        // Concurrent increase and revocation - before the message is sent.
        tsWrapper.ts.clearTestCounts();
        tsWrapper.ts.setDelayCheckAndSend(true);
        Thread updateThread = new Thread(new Runnable() {

            @Override
            public void run() {
                tsWrapper.ts.updateGuaranteedCount(1);
            }
        }, "test-update-thread");
        // This should eventually hang in the delay code.
        updateThread.start();
        // From the background thread.
        tsWrapper.ts.waitForCheckAndSendCall(1);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertFalse(ti1.getLastSetGuaranteed());
        // This won't go into checkAndSend.
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.ts.assertNoMessagesSent();
        // Release the background thread.
        tsWrapper.ts.setDelayCheckAndSend(false);
        updateThread.join();
        // No message is needed.
        tsWrapper.ts.assertNoMessagesSent();
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        // Concurrent revocation and increase - before the message is sent.
        // First, actually give it a duck.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.handleUpdateResult(ti1, true);
        tsWrapper.ts.clearTestCounts();
        assertTrue(ti1.isGuaranteed() && ti1.getLastSetGuaranteed());
        tsWrapper.ts.setDelayCheckAndSend(true);
        updateThread = new Thread(new Runnable() {

            @Override
            public void run() {
                tsWrapper.ts.updateGuaranteedCount(0);
            }
        }, "test-update-thread");
        // This should eventually hang in the delay code.
        updateThread.start();
        tsWrapper.ts.waitForCheckAndSendCall(1);
        assertFalse(ti1.isGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertTrue(ti1.getLastSetGuaranteed());
        // This won't go into checkAndSend.
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.ts.assertNoMessagesSent();
        // Release the background thread.
        tsWrapper.ts.setDelayCheckAndSend(false);
        updateThread.join();
        // No message is needed.
        tsWrapper.ts.assertNoMessagesSent();
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti1.getLastSetGuaranteed());
        assertFalse(ti1.isUpdateInProgress());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 77 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testHeartbeatInconsistency.

@Test(timeout = 10000)
public void testHeartbeatInconsistency() throws IOException, InterruptedException {
    final TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // Guaranteed flag is inconsistent based on heartbeat - another message should be send.
    try {
        Priority highPri = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1);
        assertFalse(ti1.isGuaranteed());
        // Heartbeat indicates task has a duck - this must be reverted.
        tsWrapper.ts.taskInfoUpdated(task1, true);
        tsWrapper.ts.waitForMessagesSent(1);
        assertTrue(ti1.getLastSetGuaranteed());
        assertTrue(ti1.isUpdateInProgress());
        assertFalse(ti1.isGuaranteed());
        tsWrapper.ts.handleUpdateResult(ti1, true);
        assertFalse(ti1.getLastSetGuaranteed());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 78 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testChangeGuaranteedTotal.

@Test(timeout = 10000)
public void testChangeGuaranteedTotal() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // Then revoke similarly in steps (1, 4, 1), with the opposite effect.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task3 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(0);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, highPri, new Object());
        tsWrapper.allocateTask(task2, null, lowPri, new Object());
        tsWrapper.allocateTask(task3, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(3);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1), ti2 = tsWrapper.ts.getTaskInfo(task2), ti3 = tsWrapper.ts.getTaskInfo(task3);
        assertFalse(ti1.isGuaranteed() || ti2.isGuaranteed() || ti3.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(2);
        assertTrue(ti1.isGuaranteed());
        // This particular test doesn't care which of the lower pri tasks gets the duck.
        TaskInfo ti23High = ti2.isGuaranteed() ? ti2 : ti3, ti23Low = (ti2 == ti23High) ? ti3 : ti2;
        assertTrue(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(4);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(6);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(3, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(5);
        assertTrue(ti1.isGuaranteed());
        assertTrue(ti23High.isGuaranteed());
        assertTrue(ti23Low.isGuaranteed());
        assertEquals(2, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(1);
        assertTrue(ti1.isGuaranteed());
        assertFalse(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.ts.updateGuaranteedCount(0);
        assertFalse(ti1.isGuaranteed());
        assertFalse(ti23High.isGuaranteed());
        assertFalse(ti23Low.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        tsWrapper.deallocateTask(task3, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 79 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testSimpleNoLocalityAllocation.

@Test(timeout = 10000)
public void testSimpleNoLocalityAllocation() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    try {
        Priority priority1 = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = new Object();
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, priority1, clientCookie1);
        tsWrapper.awaitTotalTaskAllocations(1);
        verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
        assertEquals(1, tsWrapper.ts.dagStats.getNumAllocationsNoLocalityRequest());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 80 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testGuaranteedScheduling.

@Test(timeout = 10000)
public void testGuaranteedScheduling() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // When the first one finishes, the duck goes to the 2nd, and then becomes unused.
    try {
        Priority priority = Priority.newInstance(1);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = new Object(), clientCookie2 = new Object();
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, priority, clientCookie1);
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti = tsWrapper.ts.getTaskInfo(task1);
        assertTrue(ti.isGuaranteed());
        assertEquals(State.ASSIGNED, ti.getState());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.allocateTask(task2, null, priority, clientCookie2);
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertFalse(ti2.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertTrue(ti2.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}