Examples with TaskStatus org.apache.druid.indexer.TaskStatus used on opensource projects

Example 41 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class TestTaskRunner method run.

@Override
public ListenableFuture<TaskStatus> run(final Task task) {
    final TaskToolbox toolbox = toolboxFactory.build(task);
    final Object taskPriorityObj = task.getContextValue(TaskThreadPriority.CONTEXT_KEY);
    int taskPriority = 0;
    try {
        taskPriority = taskPriorityObj == null ? 0 : Numbers.parseInt(taskPriorityObj);
    } catch (NumberFormatException e) {
        log.error(e, "Error parsing task priority [%s] for task [%s]", taskPriorityObj, task.getId());
    }
    final int finalTaskPriority = taskPriority;
    final ListenableFuture<TaskStatus> statusFuture = exec.computeIfAbsent(taskPriority, k -> buildExecutorService(finalTaskPriority)).submit(new TestTaskRunnerCallable(task, toolbox));
    final TestTaskRunnerWorkItem taskRunnerWorkItem = new TestTaskRunnerWorkItem(task, statusFuture);
    runningItems.add(taskRunnerWorkItem);
    Futures.addCallback(statusFuture, new FutureCallback<TaskStatus>() {

        @Override
        public void onSuccess(TaskStatus result) {
            runningItems.remove(taskRunnerWorkItem);
        }

        @Override
        public void onFailure(Throwable t) {
            runningItems.remove(taskRunnerWorkItem);
        }
    });
    return statusFuture;
}

Example 42 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class TestTaskRunner method stop.

@Override
public void stop() {
    stopping = true;
    for (Map.Entry<Integer, ListeningExecutorService> entry : exec.entrySet()) {
        try {
            entry.getValue().shutdown();
        } catch (SecurityException ex) {
            throw new RuntimeException("I can't control my own threads!", ex);
        }
    }
    for (TestTaskRunnerWorkItem item : runningItems) {
        final Task task = item.getTask();
        final long start = System.currentTimeMillis();
        if (taskConfig.isRestoreTasksOnRestart() && task.canRestore()) {
            // Attempt graceful shutdown.
            log.info("Starting graceful shutdown of task[%s].", task.getId());
            try {
                task.stopGracefully(taskConfig);
                final TaskStatus taskStatus = item.getResult().get(new Interval(DateTimes.utc(start), taskConfig.getGracefulShutdownTimeout()).toDurationMillis(), TimeUnit.MILLISECONDS);
                // Ignore status, it doesn't matter for graceful shutdowns.
                log.info("Graceful shutdown of task[%s] finished in %,dms.", task.getId(), System.currentTimeMillis() - start);
                TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), taskStatus);
            } catch (Exception e) {
                String errMsg = "Graceful shutdown of task aborted with exception, see task logs for more information";
                TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), TaskStatus.failure(task.getId(), errMsg));
                throw new RE(e, "Graceful shutdown of task[%s] aborted with exception", task.getId());
            }
        } else {
            TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), TaskStatus.failure(task.getId(), "Task failure while shutting down gracefully"));
        }
    }
    // Ok, now interrupt everything.
    for (Map.Entry<Integer, ListeningExecutorService> entry : exec.entrySet()) {
        try {
            entry.getValue().shutdownNow();
        } catch (SecurityException ex) {
            throw new RuntimeException("I can't control my own threads!", ex);
        }
    }
}

Example 43 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class RemoteTaskRunnerTest method testWorkerZKReconnect.

@Test
public void testWorkerZKReconnect() throws Exception {
    makeWorker();
    makeRemoteTaskRunner(new TestRemoteTaskRunnerConfig(new Period("PT5M")));
    Future<TaskStatus> future = remoteTaskRunner.run(task);
    Assert.assertTrue(taskAnnounced(task.getId()));
    mockWorkerRunningTask(task);
    Assert.assertTrue(workerRunningTask(task.getId()));
    byte[] bytes = cf.getData().forPath(ANNOUCEMENTS_PATH);
    cf.delete().forPath(ANNOUCEMENTS_PATH);
    // worker task cleanup scheduled
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getRemovedWorkerCleanups().containsKey(worker.getHost());
        }
    }));
    // Worker got reconnected
    cf.create().forPath(ANNOUCEMENTS_PATH, bytes);
    // worker task cleanup should get cancelled and removed
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return !remoteTaskRunner.getRemovedWorkerCleanups().containsKey(worker.getHost());
        }
    }));
    mockWorkerCompleteSuccessfulTask(task);
    TaskStatus status = future.get();
    Assert.assertEquals(status.getStatusCode(), TaskState.SUCCESS);
    Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
}

Example 44 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class RemoteTaskRunnerTest method testRunPendingTaskFailToAssignTask.

@Test
public void testRunPendingTaskFailToAssignTask() throws Exception {
    doSetup();
    RemoteTaskRunnerWorkItem originalItem = remoteTaskRunner.addPendingTask(task);
    // modify taskId to make task assignment failed
    RemoteTaskRunnerWorkItem wankyItem = Mockito.mock(RemoteTaskRunnerWorkItem.class);
    Mockito.when(wankyItem.getTaskId()).thenReturn(originalItem.getTaskId()).thenReturn("wrongId");
    remoteTaskRunner.runPendingTask(wankyItem);
    TaskStatus taskStatus = originalItem.getResult().get(0, TimeUnit.MILLISECONDS);
    Assert.assertEquals(TaskState.FAILED, taskStatus.getStatusCode());
    Assert.assertEquals("Failed to assign this task. See overlord logs for more details.", taskStatus.getErrorMsg());
}

Example 45 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class RemoteTaskRunnerTest method testSuccessfulTaskOnBlacklistedWorker.

@Test
public void testSuccessfulTaskOnBlacklistedWorker() throws Exception {
    makeWorker();
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(100);
    makeRemoteTaskRunner(rtrConfig);
    TestRealtimeTask task1 = new TestRealtimeTask("realtime1", new TaskResource("realtime1", 1), "foo", TaskStatus.success("realtime1"), jsonMapper);
    TestRealtimeTask task2 = new TestRealtimeTask("realtime2", new TaskResource("realtime2", 1), "foo", TaskStatus.success("realtime2"), jsonMapper);
    TestRealtimeTask task3 = new TestRealtimeTask("realtime3", new TaskResource("realtime3", 1), "foo", TaskStatus.success("realtime3"), jsonMapper);
    Future<TaskStatus> taskFuture1 = remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    mockWorkerCompleteFailedTask(task1);
    Assert.assertTrue(taskFuture1.get().isFailure());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture2 = remoteTaskRunner.run(task2);
    Assert.assertTrue(taskAnnounced(task2.getId()));
    mockWorkerRunningTask(task2);
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture3 = remoteTaskRunner.run(task3);
    Assert.assertTrue(taskAnnounced(task3.getId()));
    mockWorkerRunningTask(task3);
    mockWorkerCompleteFailedTask(task3);
    Assert.assertTrue(taskFuture3.get().isFailure());
    Assert.assertEquals(1, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertEquals(3, remoteTaskRunner.getBlacklistedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    mockWorkerCompleteSuccessfulTask(task2);
    Assert.assertTrue(taskFuture2.get().isSuccess());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
}