Examples with RemoteTaskRunnerConfig org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig used on opensource projects

Example 11 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class FillCapacityWithAffinityWorkerSelectStrategyTest method testFindWorkerForTaskWithNulls.

@Test
public void testFindWorkerForTaskWithNulls() {
    FillCapacityWorkerSelectStrategy strategy = new FillCapacityWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost")), false));
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost", new ImmutableWorkerInfo(new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null));
    Assert.assertEquals("lhost", worker.getWorker().getHost());
}

Example 12 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testMinCountIncrease.

@Test
public void testMinCountIncrease() {
    // Don't terminate anything
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip"));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Collections.emptyList()).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(NoopTask.create(), "http", "h1", "i1", MIN_VERSION).toImmutable())).times(3);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig()).times(2);
    EasyMock.expect(runner.getLazyWorkers()).andReturn(new ArrayList<>());
    EasyMock.expect(runner.markWorkersLazy(EasyMock.anyObject(), EasyMock.anyInt())).andReturn(Collections.emptyList());
    EasyMock.replay(runner);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean terminatedSomething = provisioner.doTerminate();
    Assert.assertFalse(terminatedSomething);
    EasyMock.verify(autoScaler);
    // Don't provision anything
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip"));
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.reset(autoScaler);
    // Increase minNumWorkers
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip"));
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("h3")));
    // Should provision two new workers
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("h4")));
    EasyMock.replay(autoScaler);
    provisionedSomething = provisioner.doProvision();
    Assert.assertTrue(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 13 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testNoActionNeeded.

@Test
public void testNoActionNeeded() {
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip"));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(Collections.singletonList((Task) NoopTask.create())).times(1);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(NoopTask.create()).toImmutable(), new TestZkWorker(NoopTask.create()).toImmutable())).times(2);
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(runner.getLazyWorkers()).andReturn(new ArrayList<>());
    EasyMock.expect(runner.markWorkersLazy(EasyMock.anyObject(), EasyMock.anyInt())).andReturn(Collections.emptyList());
    EasyMock.replay(runner);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean terminatedSomething = provisioner.doTerminate();
    Assert.assertFalse(terminatedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(1).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(Collections.singletonList("ip"));
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}

Example 14 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testProvisionNoCurrentlyRunningWorkerWithCapacityHintSetAndNoPendingTaskShouldProvisionMinimumAsCurrentIsBelowMinimum.

@Test
public void testProvisionNoCurrentlyRunningWorkerWithCapacityHintSetAndNoPendingTaskShouldProvisionMinimumAsCurrentIsBelowMinimum() {
    PendingTaskBasedWorkerProvisioningConfig config = new PendingTaskBasedWorkerProvisioningConfig().setMaxScalingDuration(new Period(1000)).setNumEventsToTrack(10).setPendingTaskTimeout(new Period(0)).setWorkerVersion(MIN_VERSION).setMaxScalingStep(2).setWorkerCapacityHint(30);
    strategy = new PendingTaskBasedWorkerProvisioningStrategy(config, DSuppliers.of(workerConfig), new ProvisioningSchedulerConfig(), new Supplier<ScheduledExecutorService>() {

        @Override
        public ScheduledExecutorService get() {
            return executorService;
        }
    });
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>());
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // No pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(new ArrayList<>());
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.emptyList());
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("aNode"))).times(3);
    EasyMock.replay(runner, autoScaler);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 3);
    for (ScalingStats.ScalingEvent event : provisioner.getStats().toList()) {
        Assert.assertTrue(event.getEvent() == ScalingStats.EVENT.PROVISION);
    }
}

Example 15 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class WorkerResourceTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(false)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(BASE_PATH);
    worker = new Worker("http", "host", "ip", 3, "v1", WorkerConfig.DEFAULT_CATEGORY);
    curatorCoordinator = new WorkerCuratorCoordinator(JSON_MAPPER, new IndexerZkConfig(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return BASE_PATH;
        }
    }, null, null, null, null), new RemoteTaskRunnerConfig(), cf, worker);
    curatorCoordinator.start();
    workerResource = new WorkerResource(worker, () -> curatorCoordinator, null, EasyMock.createNiceMock(WorkerTaskMonitor.class), ZkEnablementConfig.ENABLED);
}