Examples with RemoteTaskRunnerConfig io.druid.indexing.overlord.config.RemoteTaskRunnerConfig used on opensource projects

Example 1 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class RemoteTaskRunnerFactoryTest method testExecNotSharedBetweenRunners.

@Test
public void testExecNotSharedBetweenRunners() {
    final AtomicInteger executorCount = new AtomicInteger(0);
    RemoteTaskRunnerConfig config = new RemoteTaskRunnerConfig();
    IndexerZkConfig indexerZkConfig = new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return basePath;
        }
    }, null, null, null, null, null);
    HttpClient httpClient = EasyMock.createMock(HttpClient.class);
    Supplier<WorkerBehaviorConfig> workerBehaviorConfig = EasyMock.createMock(Supplier.class);
    ScheduledExecutorFactory executorFactory = new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int i, String s) {
            executorCount.incrementAndGet();
            return ScheduledExecutors.fixed(i, s);
        }
    };
    SimpleWorkerResourceManagementConfig resourceManagementConfig = new SimpleWorkerResourceManagementConfig();
    ResourceManagementSchedulerConfig resourceManagementSchedulerConfig = new ResourceManagementSchedulerConfig() {

        @Override
        public boolean isDoAutoscale() {
            return true;
        }
    };
    RemoteTaskRunnerFactory factory = new RemoteTaskRunnerFactory(cf, config, indexerZkConfig, jsonMapper, httpClient, workerBehaviorConfig, executorFactory, resourceManagementSchedulerConfig, new SimpleWorkerResourceManagementStrategy(resourceManagementConfig, workerBehaviorConfig, resourceManagementSchedulerConfig, executorFactory));
    Assert.assertEquals(1, executorCount.get());
    RemoteTaskRunner remoteTaskRunner1 = factory.build();
    Assert.assertEquals(2, executorCount.get());
    RemoteTaskRunner remoteTaskRunner2 = factory.build();
    Assert.assertEquals(3, executorCount.get());
}

Example 2 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class WorkerResourceTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(false)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(basePath);
    worker = new Worker("host", "ip", 3, "v1");
    curatorCoordinator = new WorkerCuratorCoordinator(jsonMapper, new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return basePath;
        }
    }, null, null, null, null, null), new RemoteTaskRunnerConfig(), cf, worker);
    curatorCoordinator.start();
    workerResource = new WorkerResource(worker, curatorCoordinator, null);
}

Example 3 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testNoActionNeeded.

@Test
public void testNoActionNeeded() throws Exception {
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList("ip"));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Arrays.asList((Task) NoopTask.create())).times(1);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(NoopTask.create()).toImmutable(), new TestZkWorker(NoopTask.create()).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(runner.getLazyWorkers()).andReturn(Lists.<Worker>newArrayList());
    EasyMock.expect(runner.markWorkersLazy((Predicate<ImmutableWorkerInfo>) EasyMock.anyObject(), EasyMock.anyInt())).andReturn(Collections.<Worker>emptyList());
    EasyMock.replay(runner);
    boolean terminatedSomething = strategy.doTerminate(runner);
    Assert.assertFalse(terminatedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList("ip"));
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 4 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testMinCountIncrease.

@Test
public void testMinCountIncrease() throws Exception {
    // Don't terminate anything
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList("ip"));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Arrays.<Task>asList()).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(NoopTask.create(), "h1", "i1", MIN_VERSION).toImmutable())).times(3);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig()).times(2);
    EasyMock.expect(runner.getLazyWorkers()).andReturn(Lists.<Worker>newArrayList());
    EasyMock.expect(runner.markWorkersLazy((Predicate<ImmutableWorkerInfo>) EasyMock.anyObject(), EasyMock.anyInt())).andReturn(Collections.<Worker>emptyList());
    EasyMock.replay(runner);
    boolean terminatedSomething = strategy.doTerminate(runner);
    Assert.assertFalse(terminatedSomething);
    EasyMock.verify(autoScaler);
    // Don't provision anything
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList("ip"));
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.reset(autoScaler);
    // Increase minNumWorkers
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList("ip"));
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("h3")));
    // Should provision two new workers
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("h4")));
    EasyMock.replay(autoScaler);
    provisionedSomething = strategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 5 with RemoteTaskRunnerConfig

use of io.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedResourceManagementStrategyTest method testSuccessfulInitialMinWorkersProvision.

@Test
public void testSuccessfulInitialMinWorkersProvision() throws Exception {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.<String>newArrayList());
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // No pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Lists.<Task>newArrayList());
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.<ImmutableWorkerInfo>asList());
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Lists.<String>newArrayList("aNode"))).times(3);
    EasyMock.replay(runner, autoScaler);
    boolean provisionedSomething = strategy.doProvision(runner);
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(strategy.getStats().toList().size() == 3);
    for (ScalingStats.ScalingEvent event : strategy.getStats().toList()) {
        Assert.assertTrue(event.getEvent() == ScalingStats.EVENT.PROVISION);
    }
}