Examples with RemoteTaskRunner org.apache.druid.indexing.overlord.RemoteTaskRunner used on opensource projects

Example 16 with RemoteTaskRunner

use of org.apache.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testProvisionWithPendingTaskAndWorkerCapacityHintSetButEmptyCurrentlyRunningWorkerShouldUseCapcityFromHintConfig.

@Test
public void testProvisionWithPendingTaskAndWorkerCapacityHintSetButEmptyCurrentlyRunningWorkerShouldUseCapcityFromHintConfig() {
    PendingTaskBasedWorkerProvisioningConfig config = new PendingTaskBasedWorkerProvisioningConfig().setMaxScalingDuration(new Period(1000)).setNumEventsToTrack(10).setPendingTaskTimeout(new Period(0)).setWorkerVersion(MIN_VERSION).setMaxScalingStep(2).setWorkerCapacityHint(30);
    strategy = new PendingTaskBasedWorkerProvisioningStrategy(config, DSuppliers.of(workerConfig), new ProvisioningSchedulerConfig(), new Supplier<ScheduledExecutorService>() {

        @Override
        public ScheduledExecutorService get() {
            return executorService;
        }
    });
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(3);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(3).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>()).times(2);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("fake"))).times(1);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // two pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(ImmutableList.of(NoopTask.create(), NoopTask.create())).times(2);
    // No currently running worker node
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.emptyList()).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig()).times(1);
    EasyMock.replay(runner);
    EasyMock.replay(autoScaler);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    // Expect to use capacity from workerCapacityHint config (which is 30)
    // and since there are two pending tasks, we will need one more worker
    Assert.assertTrue(provisionedSomething);
    Assert.assertEquals(1, provisioner.getStats().toList().size());
    DateTime createdTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertEquals(ScalingStats.EVENT.PROVISION, provisioner.getStats().toList().get(0).getEvent());
    provisionedSomething = provisioner.doProvision();
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 17 with RemoteTaskRunner

use of org.apache.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testSuccessfulInitialMinWorkersProvision.

@Test
public void testSuccessfulInitialMinWorkersProvision() {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>());
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // No pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(new ArrayList<>());
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.emptyList());
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("aNode"))).times(3);
    EasyMock.replay(runner, autoScaler);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 3);
    for (ScalingStats.ScalingEvent event : provisioner.getStats().toList()) {
        Assert.assertTrue(event.getEvent() == ScalingStats.EVENT.PROVISION);
    }
}

Example 18 with RemoteTaskRunner

use of org.apache.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testSomethingTerminating.

@Test
public void testSomethingTerminating() {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(3);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip")).times(2);
    EasyMock.expect(autoScaler.terminate(EasyMock.anyObject())).andReturn(new AutoScalingData(Collections.singletonList("ip")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getWorkers()).andReturn(ImmutableList.of(new TestZkWorker(testTask).toImmutable(), new TestZkWorker(testTask).toImmutable(), new TestZkWorker(testTask).toImmutable())).times(2);
    EasyMock.expect(runner.getLazyWorkers()).andReturn(new ArrayList<>()).times(2);
    EasyMock.expect(runner.markWorkersLazy(EasyMock.anyObject(), EasyMock.anyInt())).andReturn(Collections.singletonList(new TestZkWorker(testTask).toImmutable().getWorker()));
    EasyMock.replay(runner);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean terminatedSomething = provisioner.doTerminate();
    Assert.assertTrue(terminatedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 1);
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.TERMINATE);
    terminatedSomething = provisioner.doTerminate();
    Assert.assertFalse(terminatedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 1);
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.TERMINATE);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 19 with RemoteTaskRunner

use of org.apache.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testProvisionAlert.

@Test
public void testProvisionAlert() throws Exception {
    ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
    EmittingLogger.registerEmitter(emitter);
    emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
    EasyMock.expectLastCall();
    EasyMock.replay(emitter);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(3);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>()).times(2);
    EasyMock.expect(autoScaler.terminateWithIds(EasyMock.anyObject())).andReturn(null);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("fake")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Collections.singletonList(NoopTask.create())).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(testTask, "http", "hi", "lo", MIN_VERSION, 1).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "http", "h1", "n1", INVALID_VERSION).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "http", "h2", "n1", INVALID_VERSION).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.replay(runner);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 1);
    DateTime createdTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    Thread.sleep(2000);
    provisionedSomething = provisioner.doProvision();
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(emitter);
    EasyMock.verify(runner);
}

Example 20 with RemoteTaskRunner

use of org.apache.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testProvisionWithPendingTaskAndWorkerCapacityHintSetButNonEmptyCurrentlyRunningWorkerShouldUseCapcityFromRunningWorker.

@Test
public void testProvisionWithPendingTaskAndWorkerCapacityHintSetButNonEmptyCurrentlyRunningWorkerShouldUseCapcityFromRunningWorker() {
    PendingTaskBasedWorkerProvisioningConfig config = new PendingTaskBasedWorkerProvisioningConfig().setMaxScalingDuration(new Period(1000)).setNumEventsToTrack(10).setPendingTaskTimeout(new Period(0)).setWorkerVersion(MIN_VERSION).setMaxScalingStep(2).setWorkerCapacityHint(30);
    strategy = new PendingTaskBasedWorkerProvisioningStrategy(config, DSuppliers.of(workerConfig), new ProvisioningSchedulerConfig(), new Supplier<ScheduledExecutorService>() {

        @Override
        public ScheduledExecutorService get() {
            return executorService;
        }
    });
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(3);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(3).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>()).times(2);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("fake"))).times(2);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // two pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(ImmutableList.of(NoopTask.create(), NoopTask.create())).times(2);
    // Capacity for current worker is 1
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(testTask).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "http", "h1", "n1", INVALID_VERSION).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig()).times(1);
    EasyMock.replay(runner);
    EasyMock.replay(autoScaler);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    // Expect to use capacity from current worker (which is 1)
    // and since there are two pending tasks, we will need two more workers
    Assert.assertTrue(provisionedSomething);
    Assert.assertEquals(2, provisioner.getStats().toList().size());
    DateTime createdTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertEquals(ScalingStats.EVENT.PROVISION, provisioner.getStats().toList().get(0).getEvent());
    Assert.assertEquals(ScalingStats.EVENT.PROVISION, provisioner.getStats().toList().get(1).getEvent());
    provisionedSomething = provisioner.doProvision();
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = provisioner.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}