Examples with RemoteTaskRunner io.druid.indexing.overlord.RemoteTaskRunner used on opensource projects

Example 6 with RemoteTaskRunner

use of io.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testProvisionAlert.

@Test
public void testProvisionAlert() throws Exception {
    ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
    EmittingLogger.registerEmitter(emitter);
    emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
    EasyMock.expectLastCall();
    EasyMock.replay(emitter);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(1);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList()).times(2);
    EasyMock.expect(autoScaler.terminateWithIds(EasyMock.<List<String>>anyObject())).andReturn(null);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("fake")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Arrays.<Task>asList(NoopTask.create())).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(testTask, "hi", "lo", MIN_VERSION, 1).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "h1", "n1", INVALID_VERSION).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "h2", "n1", INVALID_VERSION).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.replay(runner);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().size() == 1);
    DateTime createdTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    Thread.sleep(2000);
    provisionedSomething = strategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(emitter);
    EasyMock.verify(runner);
}

Example 7 with RemoteTaskRunner

use of io.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class SimpleResourceManagementStrategyTest method testNullWorkerConfig.

@Test
public void testNullWorkerConfig() throws Exception {
    workerConfig.set(null);
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTasks()).andReturn(Collections.singletonList(new RemoteTaskRunnerWorkItem(testTask.getId(), null, null).withQueueInsertionTime(new DateTime()))).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(null).toImmutable())).times(1);
    EasyMock.replay(runner);
    boolean terminatedSomething = simpleResourceManagementStrategy.doTerminate(runner);
    boolean provisionedSomething = simpleResourceManagementStrategy.doProvision(runner);
    Assert.assertFalse(terminatedSomething);
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 8 with RemoteTaskRunner

use of io.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class SimpleResourceManagementStrategyTest method testProvisionAlert.

@Test
public void testProvisionAlert() throws Exception {
    ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
    EmittingLogger.registerEmitter(emitter);
    emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.replay(emitter);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2).times(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList()).times(2);
    EasyMock.expect(autoScaler.terminateWithIds(EasyMock.<List<String>>anyObject())).andReturn(null);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.newArrayList("fake")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTasks()).andReturn(Collections.singletonList(new RemoteTaskRunnerWorkItem(testTask.getId(), null, null).withQueueInsertionTime(new DateTime()))).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(testTask).toImmutable())).times(2);
    EasyMock.replay(runner);
    boolean provisionedSomething = simpleResourceManagementStrategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().size() == 1);
    DateTime createdTime = simpleResourceManagementStrategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    Thread.sleep(2000);
    provisionedSomething = simpleResourceManagementStrategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = simpleResourceManagementStrategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(emitter);
    EasyMock.verify(runner);
}

Example 9 with RemoteTaskRunner

use of io.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class SimpleResourceManagementStrategyTest method testSomethingTerminating.

@Test
public void testSomethingTerminating() throws Exception {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(1).times(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.newArrayList("ip")).times(2);
    EasyMock.expect(autoScaler.terminate(EasyMock.<List<String>>anyObject())).andReturn(new AutoScalingData(Lists.newArrayList("ip")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTasks()).andReturn(Collections.singletonList(new RemoteTaskRunnerWorkItem(testTask.getId(), null, null).withQueueInsertionTime(new DateTime()))).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(testTask).toImmutable())).times(2);
    EasyMock.expect(runner.getLazyWorkers()).andReturn(Lists.<Worker>newArrayList()).times(2);
    EasyMock.expect(runner.markWorkersLazy(EasyMock.<Predicate<ImmutableWorkerInfo>>anyObject(), EasyMock.anyInt())).andReturn(Collections.singletonList(new TestZkWorker(testTask).getWorker()));
    EasyMock.replay(runner);
    boolean terminatedSomething = simpleResourceManagementStrategy.doTerminate(runner);
    Assert.assertTrue(terminatedSomething);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().size() == 1);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.TERMINATE);
    terminatedSomething = simpleResourceManagementStrategy.doTerminate(runner);
    Assert.assertFalse(terminatedSomething);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().size() == 1);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.TERMINATE);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 10 with RemoteTaskRunner

use of io.druid.indexing.overlord.RemoteTaskRunner in project druid by druid-io.

the class SimpleResourceManagementStrategyTest method testSuccessfulProvision.

@Test
public void testSuccessfulProvision() throws Exception {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList());
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.newArrayList("aNode")));
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTasks()).andReturn(Collections.singletonList(new RemoteTaskRunnerWorkItem(testTask.getId(), null, null).withQueueInsertionTime(new DateTime())));
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(testTask).toImmutable()));
    EasyMock.replay(runner);
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = simpleResourceManagementStrategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().size() == 1);
    Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}