Examples with TezTaskAttemptID org.apache.tez.dag.records.TezTaskAttemptID used on opensource projects

Example 81 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testForcedLocalityMultiplePreemptionsSameHost1.

@Test(timeout = 10000)
@org.junit.Ignore("HIVE-25248")
public void testForcedLocalityMultiplePreemptionsSameHost1() throws IOException, InterruptedException {
    Priority priority1 = Priority.newInstance(1);
    Priority priority2 = Priority.newInstance(2);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, -1l);
    try {
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = "cookie1";
        TezTaskAttemptID task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie2 = "cookie2";
        TezTaskAttemptID task3 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie3 = "cookie3";
        TezTaskAttemptID task4 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie4 = "cookie4";
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, hostsH1, priority2, clientCookie1);
        tsWrapper.allocateTask(task2, hostsH1, priority2, clientCookie2);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        ArgumentCaptor<Container> cArgCaptor = ArgumentCaptor.forClass(Container.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), cArgCaptor.capture());
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        assertEquals(2, cArgCaptor.getAllValues().size());
        ContainerId t1CId = cArgCaptor.getAllValues().get(0).getId();
        reset(tsWrapper.mockAppCallback);
        // At this point. 2 tasks running - both at priority 2.
        // Try running a priority 1 task
        tsWrapper.allocateTask(task3, hostsH1, priority1, clientCookie3);
        while (true) {
            tsWrapper.signalSchedulerRun();
            tsWrapper.awaitSchedulerRun();
            if (tsWrapper.ts.dagStats.getNumPreemptedTasks() == 1) {
                break;
            }
        }
        ArgumentCaptor<ContainerId> cIdArgCaptor = ArgumentCaptor.forClass(ContainerId.class);
        verify(tsWrapper.mockAppCallback).preemptContainer(cIdArgCaptor.capture());
        // Determin which task has been preempted. Normally task2 would be preempted based on it starting
        // later. However - both may have the same start time, so either could be picked.
        // De-allocated now
        Object deallocatedTask1;
        // Will be de-allocated later.
        Object deallocatedTask2;
        if (cIdArgCaptor.getValue().equals(t1CId)) {
            deallocatedTask1 = task1;
            deallocatedTask2 = task2;
        } else {
            deallocatedTask1 = task2;
            deallocatedTask2 = task1;
        }
        tsWrapper.deallocateTask(deallocatedTask1, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
        tsWrapper.awaitLocalTaskAllocations(3);
        verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task3), eq(clientCookie3), any(Container.class));
        reset(tsWrapper.mockAppCallback);
        // At this point. one p=2 task and task3(p=1) running. Ask for another p1 task.
        tsWrapper.allocateTask(task4, hostsH1, priority1, clientCookie4);
        while (true) {
            tsWrapper.signalSchedulerRun();
            tsWrapper.awaitSchedulerRun();
            if (tsWrapper.ts.dagStats.getNumPreemptedTasks() == 2) {
                break;
            }
        }
        verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
        tsWrapper.deallocateTask(deallocatedTask2, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
        tsWrapper.awaitLocalTaskAllocations(4);
        verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4), eq(clientCookie4), any(Container.class));
    } finally {
        tsWrapper.shutdown();
    }
}

Example 82 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testDelayedLocalityNodeCommErrorDelayedAllocation.

@Test(timeout = 15000)
public void testDelayedLocalityNodeCommErrorDelayedAllocation() throws IOException, InterruptedException {
    Priority priority1 = Priority.newInstance(1);
    String[] hosts = new String[] { HOST1, HOST2 };
    String[] hostsH1 = new String[] { HOST1 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(5000, hosts, 1, 1, 10000l, true);
    LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled delayedTaskSchedulerCallableControlled = (LlapTaskSchedulerServiceForTestControlled.DelayedTaskSchedulerCallableControlled) tsWrapper.ts.delayedTaskSchedulerCallable;
    ControlledClock clock = tsWrapper.getClock();
    clock.setTime(clock.getTime());
    // Fill up host1 with tasks. Leave host2 empty.
    try {
        tsWrapper.controlScheduler(true);
        TezTaskAttemptID task1 = tsWrapper.allocateTask(hostsH1, priority1);
        TezTaskAttemptID task2 = tsWrapper.allocateTask(hostsH1, priority1);
        // 1 more than capacity.
        TezTaskAttemptID task3 = tsWrapper.allocateTask(hostsH1, priority1);
        tsWrapper.awaitLocalTaskAllocations(2);
        verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
        assertEquals(2, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        reset(tsWrapper.mockAppCallback);
        // Mark a task as failed due to a comm failure.
        tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.COMMUNICATION_ERROR);
        // Node1 has free capacity but is disabled. Node 2 has capcaity. Delay > re-enable tiemout
        tsWrapper.ensureNoChangeInTotalAllocations(2, 2000l);
    } finally {
        tsWrapper.shutdown();
    }
}

Example 83 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testGuaranteedTransfer.

@Test(timeout = 10000)
public void testGuaranteedTransfer() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
    // low pri task. When the high pri finishes, low pri gets the duck back.
    try {
        Priority highPri = Priority.newInstance(1), lowPri = Priority.newInstance(2);
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId(), task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        tsWrapper.ts.updateGuaranteedCount(1);
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, null, lowPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(1);
        TaskInfo ti1 = tsWrapper.ts.getTaskInfo(task1);
        assertTrue(ti1.isGuaranteed());
        assertEquals(State.ASSIGNED, ti1.getState());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.allocateTask(task2, null, highPri, new Object());
        tsWrapper.awaitTotalTaskAllocations(2);
        TaskInfo ti2 = tsWrapper.ts.getTaskInfo(task2);
        assertTrue(ti2.isGuaranteed());
        assertFalse(ti1.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task2, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertTrue(ti1.isGuaranteed());
        assertEquals(0, tsWrapper.ts.getUnusedGuaranteedCount());
        tsWrapper.deallocateTask(task1, true, TaskAttemptEndReason.CONTAINER_EXITED);
        assertEquals(1, tsWrapper.ts.getUnusedGuaranteedCount());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 84 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testNodeDisabled.

@Test(timeout = 10000)
public void testNodeDisabled() throws IOException, InterruptedException {
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(10000l);
    try {
        Priority priority1 = Priority.newInstance(1);
        String[] hosts1 = new String[] { HOST1 };
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = new Object();
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
        while (true) {
            tsWrapper.signalSchedulerRun();
            tsWrapper.awaitSchedulerRun();
            if (tsWrapper.ts.dagStats.getNumTotalAllocations() == 1) {
                break;
            }
        }
        verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
        assertEquals(1, tsWrapper.ts.dagStats.getNumLocalAllocations());
        assertEquals(0, tsWrapper.ts.dagStats.getNumAllocationsNoLocalityRequest());
        assertEquals(0, tsWrapper.ts.dagStats.getNumNonLocalAllocations());
        assertEquals(1, tsWrapper.ts.dagStats.getNumTotalAllocations());
        tsWrapper.resetAppCallback();
        tsWrapper.clock.setTime(10000l);
        tsWrapper.rejectExecution(task1);
        // Verify that the node is blacklisted
        assertEquals(1, tsWrapper.ts.dagStats.getNumRejectedTasks());
        assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
        LlapTaskSchedulerService.NodeInfo disabledNodeInfo = tsWrapper.ts.disabledNodesQueue.peek();
        assertNotNull(disabledNodeInfo);
        assertEquals(HOST1, disabledNodeInfo.getHost());
        assertEquals((10000l), disabledNodeInfo.getDelay(TimeUnit.MILLISECONDS));
        assertEquals((10000l + 10000l), disabledNodeInfo.expireTimeMillis);
        TezTaskAttemptID task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie2 = new Object();
        tsWrapper.allocateTask(task2, hosts1, priority1, clientCookie2);
        while (true) {
            tsWrapper.signalSchedulerRun();
            tsWrapper.awaitSchedulerRun();
            if (tsWrapper.ts.dagStats.getNumTotalAllocations() == 2) {
                break;
            }
        }
        verify(tsWrapper.mockAppCallback).taskAllocated(eq(task2), eq(clientCookie2), any(Container.class));
        assertEquals(1, tsWrapper.ts.dagStats.getNumLocalAllocations());
        assertEquals(0, tsWrapper.ts.dagStats.getNumAllocationsNoLocalityRequest());
        assertEquals(1, tsWrapper.ts.dagStats.getNumNonLocalAllocations());
        assertEquals(2, tsWrapper.ts.dagStats.getNumTotalAllocations());
    } finally {
        tsWrapper.shutdown();
    }
}

Example 85 with TezTaskAttemptID

use of org.apache.tez.dag.records.TezTaskAttemptID in project hive by apache.

the class TestLlapTaskSchedulerService method testHostPreferenceMissesConsistentPartialAlive.

@Test(timeout = 10000)
public void testHostPreferenceMissesConsistentPartialAlive() throws IOException, InterruptedException {
    Priority priority1 = Priority.newInstance(1);
    String[] hostsKnown = new String[] { HOST1, HOST2, HOST3, HOST4 };
    // host3 dead before scheduling
    String[] hostsLive = new String[] { HOST1, HOST2, null, HOST4 };
    String[] hostsH2 = new String[] { HOST2 };
    String[] hostsH3 = new String[] { HOST3 };
    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hostsKnown, 1, 0, 0l, false, hostsLive, true);
    try {
        TezTaskAttemptID task1 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie1 = "cookie1";
        TezTaskAttemptID task2 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie2 = "cookie2";
        TezTaskAttemptID task3 = TestTaskSchedulerServiceWrapper.generateTaskAttemptId();
        Object clientCookie3 = "cookie3";
        tsWrapper.controlScheduler(true);
        tsWrapper.allocateTask(task1, hostsH2, priority1, clientCookie1);
        tsWrapper.allocateTask(task2, hostsH2, priority1, clientCookie2);
        tsWrapper.allocateTask(task3, hostsH3, priority1, clientCookie3);
        while (true) {
            tsWrapper.signalSchedulerRun();
            tsWrapper.awaitSchedulerRun();
            if (tsWrapper.ts.dagStats.getNumTotalAllocations() == 3) {
                break;
            }
        }
        ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
        ArgumentCaptor<Container> argumentCaptor2 = ArgumentCaptor.forClass(Container.class);
        verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(argumentCaptor.capture(), any(Object.class), argumentCaptor2.capture());
        assertEquals(3, argumentCaptor.getAllValues().size());
        assertEquals(task1, argumentCaptor.getAllValues().get(0));
        assertEquals(task2, argumentCaptor.getAllValues().get(1));
        assertEquals(task3, argumentCaptor.getAllValues().get(2));
        // 1st task requested host2, got host2
        assertEquals(HOST2, argumentCaptor2.getAllValues().get(0).getNodeId().getHost());
        // 2nd task requested host2, got host4 since host3 is dead and host2 is full
        assertEquals(HOST4, argumentCaptor2.getAllValues().get(1).getNodeId().getHost());
        // 3rd task requested host3, got host1 since host3 is dead and host4 is full
        assertEquals(HOST1, argumentCaptor2.getAllValues().get(2).getNodeId().getHost());
        verify(tsWrapper.mockServiceInstanceSet, atLeast(2)).getAllInstancesOrdered(true);
        assertEquals(0, tsWrapper.ts.dagStats.getNumAllocationsNoLocalityRequest());
        assertEquals(1, tsWrapper.ts.dagStats.getNumLocalAllocations());
        assertEquals(2, tsWrapper.ts.dagStats.getNumNonLocalAllocations());
    } finally {
        tsWrapper.shutdown();
    }
}