Examples with TaskRunner org.apache.druid.indexing.overlord.TaskRunner used on opensource projects

Example 1 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class OverlordResource method getTotalWorkerCapacity.

/**
 * Gets the total worker capacity of varies states of the cluster.
 */
@GET
@Path("/totalWorkerCapacity")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(ConfigResourceFilter.class)
public Response getTotalWorkerCapacity() {
    // Calculate current cluster capacity
    int currentCapacity;
    Optional<TaskRunner> taskRunnerOptional = taskMaster.getTaskRunner();
    if (!taskRunnerOptional.isPresent()) {
        // Cannot serve call as not leader
        return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
    }
    TaskRunner taskRunner = taskRunnerOptional.get();
    Collection<ImmutableWorkerInfo> workers;
    if (taskRunner instanceof WorkerTaskRunner) {
        workers = ((WorkerTaskRunner) taskRunner).getWorkers();
        currentCapacity = workers.stream().mapToInt(workerInfo -> workerInfo.getWorker().getCapacity()).sum();
    } else {
        log.debug("Cannot calculate capacity as task runner [%s] of type [%s] does not support listing workers", taskRunner, taskRunner.getClass().getName());
        workers = ImmutableList.of();
        currentCapacity = -1;
    }
    // Calculate maximum capacity with auto scale
    int maximumCapacity;
    if (workerConfigRef == null) {
        workerConfigRef = configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class);
    }
    WorkerBehaviorConfig workerBehaviorConfig = workerConfigRef.get();
    if (workerBehaviorConfig == null) {
        // Auto scale not setup
        log.debug("Cannot calculate maximum worker capacity as worker behavior config is not configured");
        maximumCapacity = -1;
    } else if (workerBehaviorConfig instanceof DefaultWorkerBehaviorConfig) {
        DefaultWorkerBehaviorConfig defaultWorkerBehaviorConfig = (DefaultWorkerBehaviorConfig) workerBehaviorConfig;
        if (defaultWorkerBehaviorConfig.getAutoScaler() == null) {
            // Auto scale not setup
            log.debug("Cannot calculate maximum worker capacity as auto scaler not configured");
            maximumCapacity = -1;
        } else {
            int maxWorker = defaultWorkerBehaviorConfig.getAutoScaler().getMaxNumWorkers();
            int expectedWorkerCapacity = provisioningStrategy.getExpectedWorkerCapacity(workers);
            maximumCapacity = expectedWorkerCapacity == -1 ? -1 : maxWorker * expectedWorkerCapacity;
        }
    } else {
        // Auto scale is not using DefaultWorkerBehaviorConfig
        log.debug("Cannot calculate maximum worker capacity as WorkerBehaviorConfig [%s] of type [%s] does not support getting max capacity", workerBehaviorConfig, workerBehaviorConfig.getClass().getSimpleName());
        maximumCapacity = -1;
    }
    return Response.ok(new TotalWorkerCapacityResponse(currentCapacity, maximumCapacity)).build();
}

Example 2 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class KafkaSupervisorTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KafkaIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    topic = getTopic();
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    serviceEmitter = new ExceptionCapturingServiceEmitter();
    EmittingLogger.registerEmitter(serviceEmitter);
    supervisorConfig = new SupervisorStateManagerConfig();
    ingestionSchema = EasyMock.createMock(KafkaSupervisorIngestionSpec.class);
}

Example 3 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class KinesisSupervisorTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KinesisIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    supervisorRecordSupplier = createMock(KinesisRecordSupplier.class);
    tuningConfig = new KinesisSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, null, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, 5000, null, null, null, null, null, null, null, null, null);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    serviceEmitter = new ExceptionCapturingServiceEmitter();
    EmittingLogger.registerEmitter(serviceEmitter);
    supervisorConfig = new SupervisorStateManagerConfig();
}

Example 4 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class SeekableStreamSupervisor method possiblyRegisterListener.

private void possiblyRegisterListener() {
    if (listenerRegistered) {
        return;
    }
    Optional<TaskRunner> taskRunner = taskMaster.getTaskRunner();
    if (taskRunner.isPresent()) {
        taskRunner.get().registerListener(new TaskRunnerListener() {

            @Override
            public String getListenerId() {
                return supervisorId;
            }

            @Override
            public void locationChanged(final String taskId, final TaskLocation newLocation) {
            // do nothing
            }

            @Override
            public void statusChanged(String taskId, TaskStatus status) {
                addNotice(new RunNotice());
            }
        }, Execs.directExecutor());
        listenerRegistered = true;
    }
}

Example 5 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class OverlordResource method filterActiveTasks.

private List<AnyTask> filterActiveTasks(RunnerTaskState state, List<AnyTask> allTasks) {
    // divide active tasks into 3 lists : running, pending, waiting
    Optional<TaskRunner> taskRunnerOpt = taskMaster.getTaskRunner();
    if (!taskRunnerOpt.isPresent()) {
        throw new WebApplicationException(Response.serverError().entity("No task runner found").build());
    }
    TaskRunner runner = taskRunnerOpt.get();
    // counted twice but never skipped
    if (RunnerTaskState.WAITING.equals(state)) {
        Collection<? extends TaskRunnerWorkItem> runnersKnownTasks = runner.getKnownTasks();
        Set<String> runnerKnownTaskIds = runnersKnownTasks.stream().map(TaskRunnerWorkItem::getTaskId).collect(Collectors.toSet());
        final List<AnyTask> waitingTasks = new ArrayList<>();
        for (TaskRunnerWorkItem task : allTasks) {
            if (!runnerKnownTaskIds.contains(task.getTaskId())) {
                waitingTasks.add(((AnyTask) task).withTaskState(TaskState.RUNNING, RunnerTaskState.WAITING, task.getCreatedTime(), task.getQueueInsertionTime(), task.getLocation()));
            }
        }
        return waitingTasks;
    }
    if (RunnerTaskState.PENDING.equals(state)) {
        Collection<? extends TaskRunnerWorkItem> knownPendingTasks = runner.getPendingTasks();
        Set<String> pendingTaskIds = knownPendingTasks.stream().map(TaskRunnerWorkItem::getTaskId).collect(Collectors.toSet());
        Map<String, TaskRunnerWorkItem> workItemIdMap = knownPendingTasks.stream().collect(Collectors.toMap(TaskRunnerWorkItem::getTaskId, java.util.function.Function.identity(), (previousWorkItem, newWorkItem) -> newWorkItem));
        final List<AnyTask> pendingTasks = new ArrayList<>();
        for (TaskRunnerWorkItem task : allTasks) {
            if (pendingTaskIds.contains(task.getTaskId())) {
                pendingTasks.add(((AnyTask) task).withTaskState(TaskState.RUNNING, RunnerTaskState.PENDING, workItemIdMap.get(task.getTaskId()).getCreatedTime(), workItemIdMap.get(task.getTaskId()).getQueueInsertionTime(), workItemIdMap.get(task.getTaskId()).getLocation()));
            }
        }
        return pendingTasks;
    }
    if (RunnerTaskState.RUNNING.equals(state)) {
        Collection<? extends TaskRunnerWorkItem> knownRunningTasks = runner.getRunningTasks();
        Set<String> runningTaskIds = knownRunningTasks.stream().map(TaskRunnerWorkItem::getTaskId).collect(Collectors.toSet());
        Map<String, TaskRunnerWorkItem> workItemIdMap = knownRunningTasks.stream().collect(Collectors.toMap(TaskRunnerWorkItem::getTaskId, java.util.function.Function.identity(), (previousWorkItem, newWorkItem) -> newWorkItem));
        final List<AnyTask> runningTasks = new ArrayList<>();
        for (TaskRunnerWorkItem task : allTasks) {
            if (runningTaskIds.contains(task.getTaskId())) {
                runningTasks.add(((AnyTask) task).withTaskState(TaskState.RUNNING, RunnerTaskState.RUNNING, workItemIdMap.get(task.getTaskId()).getCreatedTime(), workItemIdMap.get(task.getTaskId()).getQueueInsertionTime(), workItemIdMap.get(task.getTaskId()).getLocation()));
            }
        }
        return runningTasks;
    }
    return allTasks;
}