Examples with RemoteTaskRunnerConfig io.druid.indexing.overlord.config.RemoteTaskRunnerConfig used on opensource projects

Example 6 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class RemoteTaskRunnerTest method testWorkerRemoved.

@Test
public void testWorkerRemoved() throws Exception {
    doSetup();
    Future<TaskStatus> future = remoteTaskRunner.run(task);
    Assert.assertTrue(taskAnnounced(task.getId()));
    mockWorkerRunningTask(task);
    Assert.assertTrue(workerRunningTask(task.getId()));
    cf.delete().forPath(announcementsPath);
    TaskStatus status = future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
    Assert.assertEquals(TaskStatus.Status.FAILED, status.getStatusCode());
    RemoteTaskRunnerConfig config = remoteTaskRunner.getRemoteTaskRunnerConfig();
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getRemovedWorkerCleanups().isEmpty();
        }
    }, // cleanup task is independently scheduled by event listener. we need to wait some more time.
    config.getTaskCleanupTimeout().toStandardDuration().getMillis() * 2));
    Assert.assertNull(cf.checkExists().forPath(statusPath));
}

Example 7 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testSomethingProvisioning.

@Test
public void testSomethingProvisioning() throws Exception {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(1);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList()).times(2);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("fake")));
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Arrays.<Task>asList(NoopTask.create())).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.<ImmutableWorkerInfo>asList(new TestZkWorker(testTask).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "h1", "n1", INVALID_VERSION).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig()).times(1);
    EasyMock.replay(runner);
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().size() == 1);
    DateTime createdTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    provisionedSomething = strategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 8 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testProvisionAlert.

@Test
public void testProvisionAlert() throws Exception {
    ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
    EmittingLogger.registerEmitter(emitter);
    emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
    EasyMock.expectLastCall();
    EasyMock.replay(emitter);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0).times(1);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2).times(1);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList()).times(2);
    EasyMock.expect(autoScaler.terminateWithIds(EasyMock.<List<String>>anyObject())).andReturn(null);
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("fake")));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Arrays.<Task>asList(NoopTask.create())).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(testTask, "hi", "lo", MIN_VERSION, 1).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "h1", "n1", INVALID_VERSION).toImmutable(), // Invalid version node
    new TestZkWorker(testTask, "h2", "n1", INVALID_VERSION).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.replay(runner);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().size() == 1);
    DateTime createdTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    Thread.sleep(2000);
    provisionedSomething = strategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION);
    DateTime anotherCreatedTime = strategy.getStats().toList().get(0).getTimestamp();
    Assert.assertTrue(createdTime.equals(anotherCreatedTime));
    EasyMock.verify(autoScaler);
    EasyMock.verify(emitter);
    EasyMock.verify(runner);
}

Example 9 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class EqualDistributionWorkerSelectStrategyTest method testFindWorkerForTask.

@Test
public void testFindWorkerForTask() throws Exception {
    final EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWorkerSelectStrategy();
    Optional<ImmutableWorkerInfo> optional = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("lhost", "lhost", 1, "v1"), 0, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now()), "localhost", new ImmutableWorkerInfo(new Worker("localhost", "localhost", 1, "v1"), 1, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now())), new NoopTask(null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    });
    ImmutableWorkerInfo worker = optional.get();
    Assert.assertEquals("lhost", worker.getWorker().getHost());
}

Example 10 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class EqualDistributionWorkerSelectStrategyTest method testOneDisableWorkerSameUsedCapacity.

@Test
public void testOneDisableWorkerSameUsedCapacity() throws Exception {
    String DISABLED_VERSION = "";
    final EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWorkerSelectStrategy();
    Optional<ImmutableWorkerInfo> optional = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("disableHost", "disableHost", 10, DISABLED_VERSION), 5, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now()), "localhost", new ImmutableWorkerInfo(new Worker("enableHost", "enableHost", 10, "v1"), 5, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now())), new NoopTask(null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    });
    ImmutableWorkerInfo worker = optional.get();
    Assert.assertEquals("enableHost", worker.getWorker().getHost());
}