Examples with ImmutableWorkerInfo org.apache.druid.indexing.overlord.ImmutableWorkerInfo used on opensource projects

Example 31 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class HttpRemoteTaskRunner method getIdleTaskSlotCount.

@Override
public Map<String, Long> getIdleTaskSlotCount() {
    Map<String, Long> totalIdlePeons = new HashMap<>();
    for (ImmutableWorkerInfo worker : getWorkersEligibleToRunTasks().values()) {
        String workerCategory = worker.getWorker().getCategory();
        int workerAvailableCapacity = worker.getAvailableCapacity();
        totalIdlePeons.compute(workerCategory, (category, availableCapacity) -> availableCapacity == null ? workerAvailableCapacity : availableCapacity + workerAvailableCapacity);
    }
    return totalIdlePeons;
}

Example 32 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class HttpRemoteTaskRunner method getBlacklistedTaskSlotCount.

@Override
public Map<String, Long> getBlacklistedTaskSlotCount() {
    Map<String, Long> totalBlacklistedPeons = new HashMap<>();
    for (ImmutableWorkerInfo worker : getBlackListedWorkers()) {
        String workerCategory = worker.getWorker().getCategory();
        int workerBlacklistedPeons = worker.getWorker().getCapacity();
        totalBlacklistedPeons.compute(workerCategory, (category, blacklistedPeons) -> blacklistedPeons == null ? workerBlacklistedPeons : blacklistedPeons + workerBlacklistedPeons);
    }
    return totalBlacklistedPeons;
}

Example 33 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class HttpRemoteTaskRunner method getUsedTaskSlotCount.

@Override
public Map<String, Long> getUsedTaskSlotCount() {
    Map<String, Long> totalUsedPeons = new HashMap<>();
    for (ImmutableWorkerInfo worker : getWorkers()) {
        String workerCategory = worker.getWorker().getCategory();
        int workerUsedCapacity = worker.getCurrCapacityUsed();
        totalUsedPeons.compute(workerCategory, (category, usedCapacity) -> usedCapacity == null ? workerUsedCapacity : usedCapacity + workerUsedCapacity);
    }
    return totalUsedPeons;
}

Example 34 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class WorkerSelectUtils method selectWorker.

/**
 * Helper for {@link WorkerSelectStrategy} implementations.
 *
 * @param allWorkers     map of all workers, in the style provided to {@link WorkerSelectStrategy}
 * @param affinityConfig affinity config, or null
 * @param workerSelector function that receives a list of eligible workers: version is high enough, worker can run
 *                       the task, and worker satisfies the affinity config. may return null.
 *
 * @return selected worker from "allWorkers", or null.
 */
@Nullable
public static ImmutableWorkerInfo selectWorker(final Task task, final Map<String, ImmutableWorkerInfo> allWorkers, final WorkerTaskRunnerConfig workerTaskRunnerConfig, @Nullable final AffinityConfig affinityConfig, final Function<ImmutableMap<String, ImmutableWorkerInfo>, ImmutableWorkerInfo> workerSelector) {
    final Map<String, ImmutableWorkerInfo> runnableWorkers = getRunnableWorkers(task, allWorkers, workerTaskRunnerConfig);
    if (affinityConfig == null) {
        // All runnable workers are valid.
        return workerSelector.apply(ImmutableMap.copyOf(runnableWorkers));
    } else {
        // Workers assigned to the affinity pool for our task.
        final Set<String> dataSourceWorkers = affinityConfig.getAffinity().get(task.getDataSource());
        if (dataSourceWorkers == null) {
            // No affinity config for this dataSource; use non-affinity workers.
            return workerSelector.apply(getNonAffinityWorkers(affinityConfig, runnableWorkers));
        } else {
            // Get runnable, affinity workers.
            final ImmutableMap<String, ImmutableWorkerInfo> dataSourceWorkerMap = ImmutableMap.copyOf(Maps.filterKeys(runnableWorkers, dataSourceWorkers::contains));
            final ImmutableWorkerInfo selected = workerSelector.apply(dataSourceWorkerMap);
            if (selected != null) {
                return selected;
            } else if (affinityConfig.isStrong()) {
                return null;
            } else {
                // are available.
                return workerSelector.apply(getNonAffinityWorkers(affinityConfig, runnableWorkers));
            }
        }
    }
}

Example 35 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class OverlordResourceTest method testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity.

@Test
public void testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity() {
    int invalidExpectedCapacity = -1;
    int maxNumWorkers = 2;
    WorkerTaskRunner workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class);
    Collection<ImmutableWorkerInfo> workerInfos = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "testWorker", "192.0.0.1", 3, "v1", WorkerConfig.DEFAULT_CATEGORY), 2, ImmutableSet.of("grp1", "grp2"), ImmutableSet.of("task1", "task2"), DateTimes.of("2015-01-01T01:01:01Z")));
    EasyMock.expect(workerTaskRunner.getWorkers()).andReturn(workerInfos);
    EasyMock.reset(taskMaster);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(workerTaskRunner)).anyTimes();
    EasyMock.expect(provisioningStrategy.getExpectedWorkerCapacity(workerInfos)).andReturn(invalidExpectedCapacity).anyTimes();
    AutoScaler autoScaler = EasyMock.createMock(AutoScaler.class);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(maxNumWorkers);
    DefaultWorkerBehaviorConfig workerBehaviorConfig = new DefaultWorkerBehaviorConfig(null, autoScaler);
    AtomicReference<WorkerBehaviorConfig> workerBehaviorConfigAtomicReference = new AtomicReference<>(workerBehaviorConfig);
    EasyMock.expect(configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class)).andReturn(workerBehaviorConfigAtomicReference);
    EasyMock.replay(workerTaskRunner, autoScaler, taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter, configManager, provisioningStrategy);
    final Response response = overlordResource.getTotalWorkerCapacity();
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus());
    Assert.assertEquals(workerInfos.stream().findFirst().get().getWorker().getCapacity(), ((TotalWorkerCapacityResponse) response.getEntity()).getCurrentClusterCapacity());
    Assert.assertEquals(invalidExpectedCapacity, ((TotalWorkerCapacityResponse) response.getEntity()).getMaximumCapacityWithAutoScale());
}