Examples with TaskStatus org.apache.druid.client.indexing.TaskStatus used on opensource projects

Example 1 with TaskStatus

use of org.apache.druid.client.indexing.TaskStatus in project druid by druid-io.

the class LocalIntermediaryDataManager method deleteExpiredSuprevisorTaskPartitionsIfNotRunning.

/**
 * Check supervisorTask status if its partitions have not been accessed in timeout and
 * delete all partitions for the supervisorTask if it is already finished.
 *
 * Note that the overlord sends a cleanup request when a supervisorTask is finished. The below check is to trigger
 * the self-cleanup for when the cleanup request is missing.
 */
private void deleteExpiredSuprevisorTaskPartitionsIfNotRunning() throws InterruptedException {
    final DateTime now = DateTimes.nowUtc();
    final Set<String> expiredSupervisorTasks = new HashSet<>();
    for (Entry<String, DateTime> entry : supervisorTaskCheckTimes.entrySet()) {
        final String supervisorTaskId = entry.getKey();
        final DateTime checkTime = entry.getValue();
        if (checkTime.isAfter(now)) {
            expiredSupervisorTasks.add(supervisorTaskId);
        }
    }
    if (!expiredSupervisorTasks.isEmpty()) {
        LOG.info("Found [%s] expired supervisor tasks", expiredSupervisorTasks.size());
    }
    if (!expiredSupervisorTasks.isEmpty()) {
        final Map<String, TaskStatus> taskStatuses = indexingServiceClient.getTaskStatuses(expiredSupervisorTasks);
        for (Entry<String, TaskStatus> entry : taskStatuses.entrySet()) {
            final String supervisorTaskId = entry.getKey();
            final TaskStatus status = entry.getValue();
            if (status.getStatusCode().isComplete()) {
                // If it's finished, clean up all partitions for the supervisor task.
                try {
                    deletePartitions(supervisorTaskId);
                } catch (IOException e) {
                    LOG.warn(e, "Failed to delete partitions for task[%s]", supervisorTaskId);
                }
            } else {
                // If it's still running, update last access time.
                supervisorTaskCheckTimes.put(supervisorTaskId, getExpiryTimeFromNow());
            }
        }
    }
}

Example 2 with TaskStatus

use of org.apache.druid.client.indexing.TaskStatus in project druid by druid-io.

the class LocalIntermediaryDataManagerAutoCleanupTest method setup.

@Before
public void setup() throws IOException {
    final WorkerConfig workerConfig = new WorkerConfig() {

        @Override
        public long getIntermediaryPartitionDiscoveryPeriodSec() {
            return 1;
        }

        @Override
        public long getIntermediaryPartitionCleanupPeriodSec() {
            return 2;
        }

        @Override
        public Period getIntermediaryPartitionTimeout() {
            return new Period("PT2S");
        }
    };
    final TaskConfig taskConfig = new TaskConfig(null, null, null, null, null, false, null, null, ImmutableList.of(new StorageLocationConfig(tempDir.newFolder(), null, null)), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name());
    final IndexingServiceClient indexingServiceClient = new NoopIndexingServiceClient() {

        @Override
        public Map<String, TaskStatus> getTaskStatuses(Set<String> taskIds) {
            final Map<String, TaskStatus> result = new HashMap<>();
            for (String taskId : taskIds) {
                result.put(taskId, new TaskStatus(taskId, TaskState.SUCCESS, 10));
            }
            return result;
        }
    };
    intermediaryDataManager = new LocalIntermediaryDataManager(workerConfig, taskConfig, indexingServiceClient);
    intermediaryDataManager.start();
}

Example 3 with TaskStatus

use of org.apache.druid.client.indexing.TaskStatus in project druid by druid-io.

the class ShuffleResourceTest method setup.

@Before
public void setup() throws IOException {
    final WorkerConfig workerConfig = new WorkerConfig() {

        @Override
        public long getIntermediaryPartitionDiscoveryPeriodSec() {
            return 1;
        }

        @Override
        public long getIntermediaryPartitionCleanupPeriodSec() {
            return 2;
        }

        @Override
        public Period getIntermediaryPartitionTimeout() {
            return new Period("PT2S");
        }
    };
    final TaskConfig taskConfig = new TaskConfig(null, null, null, null, null, false, null, null, ImmutableList.of(new StorageLocationConfig(tempDir.newFolder(), null, null)), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name());
    final IndexingServiceClient indexingServiceClient = new NoopIndexingServiceClient() {

        @Override
        public Map<String, TaskStatus> getTaskStatuses(Set<String> taskIds) {
            final Map<String, TaskStatus> result = new HashMap<>();
            for (String taskId : taskIds) {
                result.put(taskId, new TaskStatus(taskId, TaskState.SUCCESS, 10));
            }
            return result;
        }
    };
    intermediaryDataManager = new LocalIntermediaryDataManager(workerConfig, taskConfig, indexingServiceClient);
    shuffleMetrics = new ShuffleMetrics();
    shuffleResource = new ShuffleResource(intermediaryDataManager, Optional.of(shuffleMetrics));
}