Examples with SupervisorTaskAutoScaler org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler used on opensource projects

Example 1 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class NoopSupervisorSpecTest method testNoopSupervisorSpecWithAutoscaler.

@Test
public void testNoopSupervisorSpecWithAutoscaler() {
    Exception e = null;
    try {
        NoopSupervisorSpec noopSupervisorSpec = new NoopSupervisorSpec(null, Collections.singletonList("datasource1"));
        Supervisor supervisor = noopSupervisorSpec.createSupervisor();
        SupervisorTaskAutoScaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor);
        Assert.assertNull(autoscaler);
        Callable<Integer> noop = new Callable<Integer>() {

            @Override
            public Integer call() {
                return -1;
            }
        };
        int count = supervisor.getActiveTaskGroupsCount();
        Assert.assertEquals(count, -1);
        LagStats lagStats = supervisor.computeLagStats();
        long totalLag = lagStats.getTotalLag();
        long avgLag = lagStats.getAvgLag();
        long maxLag = lagStats.getMaxLag();
        Assert.assertEquals(totalLag, 0);
        Assert.assertEquals(avgLag, 0);
        Assert.assertEquals(maxLag, 0);
    } catch (Exception ex) {
        e = ex;
    }
    Assert.assertNull(e);
}

Example 2 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class SupervisorManager method stop.

@LifecycleStop
public void stop() {
    Preconditions.checkState(started, "SupervisorManager not started");
    synchronized (lock) {
        for (String id : supervisors.keySet()) {
            try {
                supervisors.get(id).lhs.stop(false);
                SupervisorTaskAutoScaler autoscaler = autoscalers.get(id);
                if (autoscaler != null) {
                    autoscaler.stop();
                }
            } catch (Exception e) {
                log.warn(e, "Caught exception while stopping supervisor [%s]", id);
            }
        }
        supervisors.clear();
        autoscalers.clear();
        started = false;
    }
    log.info("SupervisorManager stopped.");
}

Example 3 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class SupervisorManager method createAndStartSupervisorInternal.

/**
 * Creates a supervisor from the provided spec and starts it if there is not already a supervisor with that id.
 * <p/>
 * Caller should have acquired [lock] before invoking this method to avoid contention with other threads that may be
 * starting, stopping, suspending and resuming supervisors.
 *
 * @return true if a new supervisor was created, false if there was already an existing supervisor with this id
 */
private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean persistSpec) {
    String id = spec.getId();
    if (supervisors.containsKey(id)) {
        return false;
    }
    if (persistSpec) {
        metadataSupervisorManager.insert(id, spec);
    }
    Supervisor supervisor;
    SupervisorTaskAutoScaler autoscaler;
    try {
        supervisor = spec.createSupervisor();
        autoscaler = spec.createAutoscaler(supervisor);
        supervisor.start();
        if (autoscaler != null) {
            autoscaler.start();
            autoscalers.put(id, autoscaler);
        }
    } catch (Exception e) {
        // Supervisor creation or start failed write tombstone only when trying to start a new supervisor
        if (persistSpec) {
            metadataSupervisorManager.insert(id, new NoopSupervisorSpec(null, spec.getDataSources()));
        }
        throw new RuntimeException(e);
    }
    supervisors.put(id, Pair.of(supervisor, spec));
    return true;
}

Example 4 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class SupervisorManager method possiblyStopAndRemoveSupervisorInternal.

/**
 * Stops a supervisor with a given id and then removes it from the list.
 * <p/>
 * Caller should have acquired [lock] before invoking this method to avoid contention with other threads that may be
 * starting, stopping, suspending and resuming supervisors.
 *
 * @return true if a supervisor was stopped, false if there was no supervisor with this id
 */
private boolean possiblyStopAndRemoveSupervisorInternal(String id, boolean writeTombstone) {
    Pair<Supervisor, SupervisorSpec> pair = supervisors.get(id);
    if (pair == null) {
        return false;
    }
    if (writeTombstone) {
        metadataSupervisorManager.insert(id, new NoopSupervisorSpec(null, pair.rhs.getDataSources()));
    // where NoopSupervisorSpec is a tombstone
    }
    pair.lhs.stop(true);
    supervisors.remove(id);
    SupervisorTaskAutoScaler autoscler = autoscalers.get(id);
    if (autoscler != null) {
        autoscler.stop();
        autoscalers.remove(id);
    }
    return true;
}

Example 5 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testAutoScalerCreated.

@Test
public void testAutoScalerCreated() {
    HashMap<String, Object> autoScalerConfig = new HashMap<>();
    autoScalerConfig.put("enableTaskAutoScaler", true);
    autoScalerConfig.put("lagCollectionIntervalMillis", 500);
    autoScalerConfig.put("lagCollectionRangeMillis", 500);
    autoScalerConfig.put("scaleOutThreshold", 5000000);
    autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.3);
    autoScalerConfig.put("scaleInThreshold", 1000000);
    autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8);
    autoScalerConfig.put("scaleActionStartDelayMillis", 0);
    autoScalerConfig.put("scaleActionPeriodMillis", 100);
    autoScalerConfig.put("taskCountMax", 8);
    autoScalerConfig.put("taskCountMin", 1);
    autoScalerConfig.put("scaleInStep", 1);
    autoScalerConfig.put("scaleOutStep", 2);
    autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes();
    EasyMock.replay(ingestionSchema);
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes();
    EasyMock.replay(supervisor4);
    TestSeekableStreamSupervisorSpec spec = new TestSeekableStreamSupervisorSpec(ingestionSchema, null, false, taskStorage, taskMaster, indexerMetadataStorageCoordinator, indexTaskClientFactory, mapper, emitter, monitorSchedulerConfig, rowIngestionMetersFactory, supervisorStateManagerConfig, supervisor4, "id1");
    SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler instanceof LagBasedAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.put("enableTaskAutoScaler", false);
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    SupervisorTaskAutoScaler autoscaler2 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler2 instanceof NoopTaskAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.remove("enableTaskAutoScaler");
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    SupervisorTaskAutoScaler autoscaler3 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler3 instanceof NoopTaskAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.clear();
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    Assert.assertTrue(autoScalerConfig.isEmpty());
    SupervisorTaskAutoScaler autoscaler4 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler4 instanceof NoopTaskAutoScaler);
}