Examples with LogicalIOProcessorRuntimeTask org.apache.tez.runtime.LogicalIOProcessorRuntimeTask used on opensource projects

Example 6 with LogicalIOProcessorRuntimeTask

use of org.apache.tez.runtime.LogicalIOProcessorRuntimeTask in project tez by apache.

the class TestTaskExecution2 method testMultipleSuccessfulTasks.

@Test(timeout = 5000)
public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY, true);
        LogicalIOProcessorRuntimeTask runtimeTask = taskRunner.task;
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        assertFalse(TestProcessor.wasAborted());
        umbilical.resetTrackedEvents();
        TezCounters tezCounters = runtimeTask.getCounters();
        verifySysCounters(tezCounters, 5, 5);
        taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY, false);
        runtimeTask = taskRunner.task;
        // Setup the executor
        taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        assertFalse(TestProcessor.wasAborted());
        tezCounters = runtimeTask.getCounters();
        verifySysCounters(tezCounters, -1, -1);
    } finally {
        executor.shutdownNow();
    }
}

Example 7 with LogicalIOProcessorRuntimeTask

use of org.apache.tez.runtime.LogicalIOProcessorRuntimeTask in project tez by apache.

the class TestProcessorContext method testDagNumber.

@Test(timeout = 5000)
public void testDagNumber() throws IOException {
    String[] localDirs = new String[] { "dummyLocalDir" };
    int appAttemptNumber = 1;
    TezUmbilical tezUmbilical = mock(TezUmbilical.class);
    String dagName = "DAG_NAME";
    String vertexName = "VERTEX_NAME";
    int vertexParallelism = 20;
    int dagNumber = 52;
    ApplicationId appId = ApplicationId.newInstance(10000, 13);
    TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 6);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 4);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2);
    TaskSpec mockSpec = mock(TaskSpec.class);
    when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class)));
    when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class)));
    Configuration conf = new Configuration();
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(conf);
    LogicalIOProcessorRuntimeTask runtimeTask = new LogicalIOProcessorRuntimeTask(mockSpec, 1, conf, new String[] { "/" }, tezUmbilical, null, null, null, null, "", null, 1024, false, new DefaultHadoopShim(), sharedExecutor);
    LogicalIOProcessorRuntimeTask mockTask = spy(runtimeTask);
    Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap();
    Map<String, String> auxServiceEnv = Maps.newHashMap();
    MemoryDistributor memDist = mock(MemoryDistributor.class);
    ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class);
    InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class);
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    ExecutionContext execContext = new ExecutionContextImpl("localhost");
    long memAvailable = 10000l;
    TezProcessorContextImpl procContext = new TezProcessorContextImpl(new Configuration(), localDirs, appAttemptNumber, tezUmbilical, dagName, vertexName, vertexParallelism, taskAttemptId, null, runtimeTask, serviceConsumerMetadata, auxServiceEnv, memDist, processorDesc, inputReadyTracker, objectRegistry, execContext, memAvailable, sharedExecutor);
    assertEquals(dagNumber, procContext.getDagIdentifier());
    assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber());
    assertEquals(appId, procContext.getApplicationId());
    assertEquals(dagName, procContext.getDAGName());
    assertEquals(vertexName, procContext.getTaskVertexName());
    assertEquals(vertexId.getId(), procContext.getTaskVertexIndex());
    assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs()));
    // test auto call of notifyProgress
    procContext.setProgress(0.1f);
    verify(mockTask, times(1)).notifyProgressInvocation();
    sharedExecutor.shutdown();
}

Example 8 with LogicalIOProcessorRuntimeTask

use of org.apache.tez.runtime.LogicalIOProcessorRuntimeTask in project tez by apache.

the class TestTaskReporter method testContinuousHeartbeatsOnMaxEvents.

@Test(timeout = 10000)
public void testContinuousHeartbeatsOnMaxEvents() throws Exception {
    final Object lock = new Object();
    final AtomicBoolean hb2Done = new AtomicBoolean(false);
    TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            TezHeartbeatRequest request = (TezHeartbeatRequest) args[0];
            if (request.getRequestId() == 1 || request.getRequestId() == 2) {
                TezHeartbeatResponse response = new TezHeartbeatResponse(createEvents(5));
                response.setLastRequestId(request.getRequestId());
                return response;
            } else if (request.getRequestId() == 3) {
                TezHeartbeatResponse response = new TezHeartbeatResponse(createEvents(1));
                response.setLastRequestId(request.getRequestId());
                synchronized (lock) {
                    hb2Done.set(true);
                    lock.notify();
                }
                return response;
            } else {
                throw new TezUncheckedException("Invalid request id for test: " + request.getRequestId());
            }
        }
    }).when(mockUmbilical).heartbeat(any(TezHeartbeatRequest.class));
    TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
    LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
    doReturn("vertexName").when(mockTask).getVertexName();
    doReturn(mockTaskAttemptId).when(mockTask).getTaskAttemptID();
    // Setup the sleep time to be way higher than the test timeout
    TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
    ExecutorService executor = Executors.newSingleThreadExecutor();
    executor.submit(heartbeatCallable);
    try {
        synchronized (lock) {
            if (!hb2Done.get()) {
                lock.wait();
            }
        }
        verify(mockUmbilical, times(3)).heartbeat(any(TezHeartbeatRequest.class));
        Thread.sleep(2000l);
        // Sleep for 2 seconds, less than the callable sleep time. No more invocations.
        verify(mockUmbilical, times(3)).heartbeat(any(TezHeartbeatRequest.class));
    } finally {
        executor.shutdownNow();
    }
}

Example 9 with LogicalIOProcessorRuntimeTask

use of org.apache.tez.runtime.LogicalIOProcessorRuntimeTask in project tez by apache.

the class TestTaskReporter method testEventThrottling.

@Test(timeout = 10000)
public void testEventThrottling() throws Exception {
    TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
    LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
    when(mockTask.getMaxEventsToHandle()).thenReturn(10000, 1);
    when(mockTask.getVertexName()).thenReturn("vertexName");
    when(mockTask.getTaskAttemptID()).thenReturn(mockTaskAttemptId);
    TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
    TezHeartbeatResponse resp1 = new TezHeartbeatResponse(createEvents(5));
    resp1.setLastRequestId(1);
    TezHeartbeatResponse resp2 = new TezHeartbeatResponse(createEvents(1));
    resp2.setLastRequestId(2);
    resp2.setShouldDie();
    when(mockUmbilical.heartbeat(isA(TezHeartbeatRequest.class))).thenReturn(resp1, resp2);
    // Setup the sleep time to be way higher than the test timeout
    TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
    ExecutorService executor = Executors.newSingleThreadExecutor();
    try {
        Future<Boolean> result = executor.submit(heartbeatCallable);
        Assert.assertFalse(result.get());
    } finally {
        executor.shutdownNow();
    }
    ArgumentCaptor<TezHeartbeatRequest> captor = ArgumentCaptor.forClass(TezHeartbeatRequest.class);
    verify(mockUmbilical, times(2)).heartbeat(captor.capture());
    TezHeartbeatRequest req = captor.getValue();
    Assert.assertEquals(2, req.getRequestId());
    Assert.assertEquals(1, req.getMaxEvents());
}

Example 10 with LogicalIOProcessorRuntimeTask

use of org.apache.tez.runtime.LogicalIOProcessorRuntimeTask in project tez by apache.

the class TestTaskReporter method testStatusUpdateAfterInitializationAndCounterFlag.

@Test(timeout = 5000)
public void testStatusUpdateAfterInitializationAndCounterFlag() {
    TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
    LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
    doReturn("vertexName").when(mockTask).getVertexName();
    doReturn(mockTaskAttemptId).when(mockTask).getTaskAttemptID();
    boolean progressNotified = false;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
    float progress = 0.5f;
    TaskStatistics stats = new TaskStatistics();
    TezCounters counters = new TezCounters();
    doReturn(progress).when(mockTask).getProgress();
    doReturn(stats).when(mockTask).getTaskStatistics();
    doReturn(counters).when(mockTask).getCounters();
    // Setup the sleep time to be way higher than the test timeout
    TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
    // task not initialized - nothing obtained from task
    doReturn(false).when(mockTask).hasInitialized();
    TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(1)).hasInitialized();
    verify(mockTask, times(0)).getProgress();
    verify(mockTask, times(0)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(0, event.getProgress(), 0);
    Assert.assertEquals(false, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained but not counters since flag is false
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(false);
    verify(mockTask, times(2)).hasInitialized();
    verify(mockTask, times(1)).getProgress();
    verify(mockTask, times(1)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained and also counters since flag is true
    progressNotified = true;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(3)).hasInitialized();
    verify(mockTask, times(2)).getProgress();
    verify(mockTask, times(2)).getAndClearProgressNotification();
    verify(mockTask, times(1)).getTaskStatistics();
    verify(mockTask, times(1)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertEquals(counters, event.getCounters());
    Assert.assertEquals(stats, event.getStatistics());
}