Examples with ImmutableWorkerInfo org.apache.druid.indexing.overlord.ImmutableWorkerInfo used on opensource projects

Example 1 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class OverlordResource method getTotalWorkerCapacity.

/**
 * Gets the total worker capacity of varies states of the cluster.
 */
@GET
@Path("/totalWorkerCapacity")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(ConfigResourceFilter.class)
public Response getTotalWorkerCapacity() {
    // Calculate current cluster capacity
    int currentCapacity;
    Optional<TaskRunner> taskRunnerOptional = taskMaster.getTaskRunner();
    if (!taskRunnerOptional.isPresent()) {
        // Cannot serve call as not leader
        return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
    }
    TaskRunner taskRunner = taskRunnerOptional.get();
    Collection<ImmutableWorkerInfo> workers;
    if (taskRunner instanceof WorkerTaskRunner) {
        workers = ((WorkerTaskRunner) taskRunner).getWorkers();
        currentCapacity = workers.stream().mapToInt(workerInfo -> workerInfo.getWorker().getCapacity()).sum();
    } else {
        log.debug("Cannot calculate capacity as task runner [%s] of type [%s] does not support listing workers", taskRunner, taskRunner.getClass().getName());
        workers = ImmutableList.of();
        currentCapacity = -1;
    }
    // Calculate maximum capacity with auto scale
    int maximumCapacity;
    if (workerConfigRef == null) {
        workerConfigRef = configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class);
    }
    WorkerBehaviorConfig workerBehaviorConfig = workerConfigRef.get();
    if (workerBehaviorConfig == null) {
        // Auto scale not setup
        log.debug("Cannot calculate maximum worker capacity as worker behavior config is not configured");
        maximumCapacity = -1;
    } else if (workerBehaviorConfig instanceof DefaultWorkerBehaviorConfig) {
        DefaultWorkerBehaviorConfig defaultWorkerBehaviorConfig = (DefaultWorkerBehaviorConfig) workerBehaviorConfig;
        if (defaultWorkerBehaviorConfig.getAutoScaler() == null) {
            // Auto scale not setup
            log.debug("Cannot calculate maximum worker capacity as auto scaler not configured");
            maximumCapacity = -1;
        } else {
            int maxWorker = defaultWorkerBehaviorConfig.getAutoScaler().getMaxNumWorkers();
            int expectedWorkerCapacity = provisioningStrategy.getExpectedWorkerCapacity(workers);
            maximumCapacity = expectedWorkerCapacity == -1 ? -1 : maxWorker * expectedWorkerCapacity;
        }
    } else {
        // Auto scale is not using DefaultWorkerBehaviorConfig
        log.debug("Cannot calculate maximum worker capacity as WorkerBehaviorConfig [%s] of type [%s] does not support getting max capacity", workerBehaviorConfig, workerBehaviorConfig.getClass().getSimpleName());
        maximumCapacity = -1;
    }
    return Response.ok(new TotalWorkerCapacityResponse(currentCapacity, maximumCapacity)).build();
}

Example 2 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class WorkerSelectUtils method selectWorker.

/**
 * Helper for {@link WorkerSelectStrategy} implementations.
 *
 * @param allWorkers     map of all workers, in the style provided to {@link WorkerSelectStrategy}
 * @param workerCategorySpec worker category spec, or null
 * @param workerSelector function that receives a list of eligible workers: version is high enough, worker can run
 *                       the task, and worker satisfies the worker category spec. may return null.
 *
 * @return selected worker from "allWorkers", or null.
 */
@Nullable
public static ImmutableWorkerInfo selectWorker(final Task task, final Map<String, ImmutableWorkerInfo> allWorkers, final WorkerTaskRunnerConfig workerTaskRunnerConfig, @Nullable final WorkerCategorySpec workerCategorySpec, final Function<ImmutableMap<String, ImmutableWorkerInfo>, ImmutableWorkerInfo> workerSelector) {
    final Map<String, ImmutableWorkerInfo> runnableWorkers = getRunnableWorkers(task, allWorkers, workerTaskRunnerConfig);
    // select worker according to worker category spec
    if (workerCategorySpec != null) {
        final WorkerCategorySpec.CategoryConfig categoryConfig = workerCategorySpec.getCategoryMap().get(task.getType());
        if (categoryConfig != null) {
            final String defaultCategory = categoryConfig.getDefaultCategory();
            final Map<String, String> categoryAffinity = categoryConfig.getCategoryAffinity();
            String preferredCategory = categoryAffinity.get(task.getDataSource());
            // If there is no preferred category for the datasource, then using the defaultCategory. However, the defaultCategory
            // may be null too, so we need to do one more null check (see below).
            preferredCategory = preferredCategory == null ? defaultCategory : preferredCategory;
            if (preferredCategory != null) {
                // select worker from preferred category
                final ImmutableMap<String, ImmutableWorkerInfo> categoryWorkers = getCategoryWorkers(preferredCategory, runnableWorkers);
                final ImmutableWorkerInfo selected = workerSelector.apply(categoryWorkers);
                if (selected != null) {
                    return selected;
                } else if (workerCategorySpec.isStrong()) {
                    return null;
                }
            }
        }
    }
    // select worker from all runnable workers by default
    return workerSelector.apply(ImmutableMap.copyOf(runnableWorkers));
}

Example 3 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class EqualDistributionWithAffinityWorkerSelectStrategyTest method testIsolation.

@Test
public void testIsolation() {
    EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost")), false));
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost", new ImmutableWorkerInfo(new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null));
    Assert.assertNull(worker);
}

Example 4 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class EqualDistributionWithAffinityWorkerSelectStrategyTest method testFindWorkerForTask.

@Test
public void testFindWorkerForTask() {
    EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost1", "localhost2", "localhost3")), false));
    NoopTask noopTask = new NoopTask(null, null, null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    };
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost0", new ImmutableWorkerInfo(new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost1", new ImmutableWorkerInfo(new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost2", new ImmutableWorkerInfo(new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost3", new ImmutableWorkerInfo(new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), noopTask);
    Assert.assertEquals("localhost1", worker.getWorker().getHost());
}

Example 5 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class EqualDistributionWorkerSelectStrategyTest method testOneDisableWorkerDifferentUsedCapacity.

@Test
public void testOneDisableWorkerDifferentUsedCapacity() {
    String disabledVersion = "";
    final EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWorkerSelectStrategy(null);
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("http", "disableHost", "disableHost", 10, disabledVersion, WorkerConfig.DEFAULT_CATEGORY), 2, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost", new ImmutableWorkerInfo(new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    });
    Assert.assertEquals("enableHost", worker.getWorker().getHost());
}