Examples with LagStats org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats used on opensource projects

Example 1 with LagStats

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats in project druid by druid-io.

the class NoopSupervisorSpecTest method testNoopSupervisorSpecWithAutoscaler.

@Test
public void testNoopSupervisorSpecWithAutoscaler() {
    Exception e = null;
    try {
        NoopSupervisorSpec noopSupervisorSpec = new NoopSupervisorSpec(null, Collections.singletonList("datasource1"));
        Supervisor supervisor = noopSupervisorSpec.createSupervisor();
        SupervisorTaskAutoScaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor);
        Assert.assertNull(autoscaler);
        Callable<Integer> noop = new Callable<Integer>() {

            @Override
            public Integer call() {
                return -1;
            }
        };
        int count = supervisor.getActiveTaskGroupsCount();
        Assert.assertEquals(count, -1);
        LagStats lagStats = supervisor.computeLagStats();
        long totalLag = lagStats.getTotalLag();
        long avgLag = lagStats.getAvgLag();
        long maxLag = lagStats.getMaxLag();
        Assert.assertEquals(totalLag, 0);
        Assert.assertEquals(avgLag, 0);
        Assert.assertEquals(maxLag, 0);
    } catch (Exception ex) {
        e = ex;
    }
    Assert.assertNull(e);
}

Example 2 with LagStats

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testSeekableStreamSupervisorSpecWithScaleOutSmallPartitionNumber.

@Test
public void testSeekableStreamSupervisorSpecWithScaleOutSmallPartitionNumber() throws InterruptedException {
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig(1, true)).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes();
    EasyMock.replay(spec);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes();
    EasyMock.replay(ingestionSchema);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes();
    EasyMock.replay(taskMaster);
    TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(2);
    LagStats lagStats = supervisor.computeLagStats();
    LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(3), LagBasedAutoScalerConfig.class), spec);
    supervisor.start();
    autoScaler.start();
    supervisor.runInternal();
    int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScaleOut);
    Thread.sleep(1 * 1000);
    int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountAfterScaleOut);
    autoScaler.reset();
    autoScaler.stop();
}

Example 3 with LagStats

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testSeekableStreamSupervisorSpecWithScaleOut.

@Test
public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedException {
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig(1, true)).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes();
    EasyMock.replay(spec);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes();
    EasyMock.replay(ingestionSchema);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes();
    EasyMock.replay(taskMaster);
    TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(3);
    LagStats lagStats = supervisor.computeLagStats();
    LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(2), LagBasedAutoScalerConfig.class), spec);
    supervisor.start();
    autoScaler.start();
    supervisor.runInternal();
    int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScaleOut);
    Thread.sleep(1 * 1000);
    int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountAfterScaleOut);
    autoScaler.reset();
    autoScaler.stop();
}