Examples with TaskState org.apache.druid.indexer.TaskState used on opensource projects

Example 6 with TaskState

use of org.apache.druid.indexer.TaskState in project druid by druid-io.

the class ParallelIndexPhaseRunner method run.

@Override
public TaskState run() throws Exception {
    final CountingSubTaskSpecIterator subTaskSpecIterator = new CountingSubTaskSpecIterator(subTaskSpecIterator());
    if (!subTaskSpecIterator.hasNext()) {
        LOG.warn("There's no input split to process");
        return TaskState.SUCCESS;
    }
    final long taskStatusCheckingPeriod = tuningConfig.getTaskStatusCheckPeriodMs();
    taskMonitor = new TaskMonitor<>(toolbox.getIndexingServiceClient(), tuningConfig.getMaxRetry(), estimateTotalNumSubTasks());
    TaskState state = TaskState.RUNNING;
    taskMonitor.start(taskStatusCheckingPeriod);
    try {
        LOG.info("Submitting initial tasks");
        // Submit initial tasks
        while (isRunning() && subTaskSpecIterator.hasNext() && taskMonitor.getNumRunningTasks() < maxNumConcurrentSubTasks) {
            submitNewTask(taskMonitor, subTaskSpecIterator.next());
        }
        LOG.info("Waiting for subTasks to be completed");
        while (isRunning()) {
            final SubTaskCompleteEvent<SubTaskType> taskCompleteEvent = taskCompleteEvents.poll(taskStatusCheckingPeriod, TimeUnit.MILLISECONDS);
            if (taskCompleteEvent != null) {
                final TaskState completeState = taskCompleteEvent.getLastState();
                getSubtaskCompletionCallback(taskCompleteEvent).run();
                switch(completeState) {
                    case SUCCESS:
                        final TaskStatusPlus completeStatus = taskCompleteEvent.getLastStatus();
                        if (completeStatus == null) {
                            throw new ISE("Last status of complete task is missing!");
                        }
                        if (!subTaskSpecIterator.hasNext()) {
                            // We have no more subTasks to run
                            if (taskMonitor.getNumRunningTasks() == 0 && taskCompleteEvents.isEmpty()) {
                                subTaskScheduleAndMonitorStopped = true;
                                if (subTaskSpecIterator.count == taskMonitor.getNumSucceededTasks()) {
                                    // Succeeded
                                    state = TaskState.SUCCESS;
                                } else {
                                    // Failed
                                    final ParallelIndexingPhaseProgress monitorStatus = taskMonitor.getProgress();
                                    throw new ISE("Expected [%d] tasks to succeed, but we got [%d] succeeded tasks and [%d] failed tasks", subTaskSpecIterator.count, monitorStatus.getSucceeded(), monitorStatus.getFailed());
                                }
                            }
                        } else if (taskMonitor.getNumRunningTasks() < maxNumConcurrentSubTasks) {
                            // We have more subTasks to run
                            submitNewTask(taskMonitor, subTaskSpecIterator.next());
                        } else {
                        // We have more subTasks to run, but don't have enough available task slots
                        // do nothing
                        }
                        break;
                    case FAILED:
                        // TaskMonitor already tried everything it can do for failed tasks. We failed.
                        state = TaskState.FAILED;
                        subTaskScheduleAndMonitorStopped = true;
                        final TaskStatusPlus lastStatus = taskCompleteEvent.getLastStatus();
                        if (lastStatus != null) {
                            LOG.error("Failed because of the failed sub task[%s]", lastStatus.getId());
                        } else {
                            final SubTaskSpec<?> spec = taskCompleteEvent.getSpec();
                            LOG.error("Failed to process spec[%s] with an unknown last status", spec.getId());
                        }
                        break;
                    default:
                        throw new ISE("spec[%s] is in an invalid state[%s]", taskCompleteEvent.getSpec().getId(), completeState);
                }
            }
        }
    } finally {
        stopInternal();
        if (!state.isComplete()) {
            state = TaskState.FAILED;
        }
    }
    return state;
}