Examples with WorkerTaskRunner org.apache.druid.indexing.overlord.WorkerTaskRunner used on opensource projects

Example 1 with WorkerTaskRunner

use of org.apache.druid.indexing.overlord.WorkerTaskRunner in project druid by druid-io.

the class OverlordResource method getTotalWorkerCapacity.

/**
 * Gets the total worker capacity of varies states of the cluster.
 */
@GET
@Path("/totalWorkerCapacity")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(ConfigResourceFilter.class)
public Response getTotalWorkerCapacity() {
    // Calculate current cluster capacity
    int currentCapacity;
    Optional<TaskRunner> taskRunnerOptional = taskMaster.getTaskRunner();
    if (!taskRunnerOptional.isPresent()) {
        // Cannot serve call as not leader
        return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
    }
    TaskRunner taskRunner = taskRunnerOptional.get();
    Collection<ImmutableWorkerInfo> workers;
    if (taskRunner instanceof WorkerTaskRunner) {
        workers = ((WorkerTaskRunner) taskRunner).getWorkers();
        currentCapacity = workers.stream().mapToInt(workerInfo -> workerInfo.getWorker().getCapacity()).sum();
    } else {
        log.debug("Cannot calculate capacity as task runner [%s] of type [%s] does not support listing workers", taskRunner, taskRunner.getClass().getName());
        workers = ImmutableList.of();
        currentCapacity = -1;
    }
    // Calculate maximum capacity with auto scale
    int maximumCapacity;
    if (workerConfigRef == null) {
        workerConfigRef = configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class);
    }
    WorkerBehaviorConfig workerBehaviorConfig = workerConfigRef.get();
    if (workerBehaviorConfig == null) {
        // Auto scale not setup
        log.debug("Cannot calculate maximum worker capacity as worker behavior config is not configured");
        maximumCapacity = -1;
    } else if (workerBehaviorConfig instanceof DefaultWorkerBehaviorConfig) {
        DefaultWorkerBehaviorConfig defaultWorkerBehaviorConfig = (DefaultWorkerBehaviorConfig) workerBehaviorConfig;
        if (defaultWorkerBehaviorConfig.getAutoScaler() == null) {
            // Auto scale not setup
            log.debug("Cannot calculate maximum worker capacity as auto scaler not configured");
            maximumCapacity = -1;
        } else {
            int maxWorker = defaultWorkerBehaviorConfig.getAutoScaler().getMaxNumWorkers();
            int expectedWorkerCapacity = provisioningStrategy.getExpectedWorkerCapacity(workers);
            maximumCapacity = expectedWorkerCapacity == -1 ? -1 : maxWorker * expectedWorkerCapacity;
        }
    } else {
        // Auto scale is not using DefaultWorkerBehaviorConfig
        log.debug("Cannot calculate maximum worker capacity as WorkerBehaviorConfig [%s] of type [%s] does not support getting max capacity", workerBehaviorConfig, workerBehaviorConfig.getClass().getSimpleName());
        maximumCapacity = -1;
    }
    return Response.ok(new TotalWorkerCapacityResponse(currentCapacity, maximumCapacity)).build();
}

Example 2 with WorkerTaskRunner

use of org.apache.druid.indexing.overlord.WorkerTaskRunner in project druid by druid-io.

the class OverlordResourceTest method testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategySupportExpectedWorkerCapacity.

@Test
public void testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategySupportExpectedWorkerCapacity() {
    int expectedWorkerCapacity = 3;
    int maxNumWorkers = 2;
    WorkerTaskRunner workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class);
    Collection<ImmutableWorkerInfo> workerInfos = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "testWorker", "192.0.0.1", expectedWorkerCapacity, "v1", WorkerConfig.DEFAULT_CATEGORY), 2, ImmutableSet.of("grp1", "grp2"), ImmutableSet.of("task1", "task2"), DateTimes.of("2015-01-01T01:01:01Z")));
    EasyMock.expect(workerTaskRunner.getWorkers()).andReturn(workerInfos);
    EasyMock.reset(taskMaster);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(workerTaskRunner)).anyTimes();
    EasyMock.expect(provisioningStrategy.getExpectedWorkerCapacity(workerInfos)).andReturn(expectedWorkerCapacity).anyTimes();
    AutoScaler autoScaler = EasyMock.createMock(AutoScaler.class);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(maxNumWorkers);
    DefaultWorkerBehaviorConfig workerBehaviorConfig = new DefaultWorkerBehaviorConfig(null, autoScaler);
    AtomicReference<WorkerBehaviorConfig> workerBehaviorConfigAtomicReference = new AtomicReference<>(workerBehaviorConfig);
    EasyMock.expect(configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class)).andReturn(workerBehaviorConfigAtomicReference);
    EasyMock.replay(workerTaskRunner, autoScaler, taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter, configManager, provisioningStrategy);
    final Response response = overlordResource.getTotalWorkerCapacity();
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus());
    Assert.assertEquals(expectedWorkerCapacity, ((TotalWorkerCapacityResponse) response.getEntity()).getCurrentClusterCapacity());
    Assert.assertEquals(expectedWorkerCapacity * maxNumWorkers, ((TotalWorkerCapacityResponse) response.getEntity()).getMaximumCapacityWithAutoScale());
}

Example 3 with WorkerTaskRunner

use of org.apache.druid.indexing.overlord.WorkerTaskRunner in project druid by druid-io.

the class OverlordResourceTest method testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity.

@Test
public void testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity() {
    int invalidExpectedCapacity = -1;
    int maxNumWorkers = 2;
    WorkerTaskRunner workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class);
    Collection<ImmutableWorkerInfo> workerInfos = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "testWorker", "192.0.0.1", 3, "v1", WorkerConfig.DEFAULT_CATEGORY), 2, ImmutableSet.of("grp1", "grp2"), ImmutableSet.of("task1", "task2"), DateTimes.of("2015-01-01T01:01:01Z")));
    EasyMock.expect(workerTaskRunner.getWorkers()).andReturn(workerInfos);
    EasyMock.reset(taskMaster);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(workerTaskRunner)).anyTimes();
    EasyMock.expect(provisioningStrategy.getExpectedWorkerCapacity(workerInfos)).andReturn(invalidExpectedCapacity).anyTimes();
    AutoScaler autoScaler = EasyMock.createMock(AutoScaler.class);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(maxNumWorkers);
    DefaultWorkerBehaviorConfig workerBehaviorConfig = new DefaultWorkerBehaviorConfig(null, autoScaler);
    AtomicReference<WorkerBehaviorConfig> workerBehaviorConfigAtomicReference = new AtomicReference<>(workerBehaviorConfig);
    EasyMock.expect(configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class)).andReturn(workerBehaviorConfigAtomicReference);
    EasyMock.replay(workerTaskRunner, autoScaler, taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter, configManager, provisioningStrategy);
    final Response response = overlordResource.getTotalWorkerCapacity();
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus());
    Assert.assertEquals(workerInfos.stream().findFirst().get().getWorker().getCapacity(), ((TotalWorkerCapacityResponse) response.getEntity()).getCurrentClusterCapacity());
    Assert.assertEquals(invalidExpectedCapacity, ((TotalWorkerCapacityResponse) response.getEntity()).getMaximumCapacityWithAutoScale());
}