Examples with RemoteTaskRunnerConfig org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig used on opensource projects

Example 1 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class RemoteTaskRunnerTest method testBootstrap.

@Test
public void testBootstrap() throws Exception {
    makeWorker();
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(100);
    makeRemoteTaskRunner(rtrConfig);
    TestRealtimeTask task1 = new TestRealtimeTask("first", new TaskResource("first", 1), "foo", TaskStatus.running("first"), jsonMapper);
    remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    TestRealtimeTask task = new TestRealtimeTask("second", new TaskResource("task", 2), "foo", TaskStatus.running("task"), jsonMapper);
    remoteTaskRunner.run(task);
    TestRealtimeTask task2 = new TestRealtimeTask("second", new TaskResource("second", 2), "foo", TaskStatus.running("second"), jsonMapper);
    remoteTaskRunner.run(task2);
    Assert.assertTrue(taskAnnounced(task2.getId()));
    mockWorkerRunningTask(task2);
    final Set<String> runningTasks = Sets.newHashSet(Iterables.transform(remoteTaskRunner.getRunningTasks(), new Function<RemoteTaskRunnerWorkItem, String>() {

        @Override
        public String apply(RemoteTaskRunnerWorkItem input) {
            return input.getTaskId();
        }
    }));
    Assert.assertEquals("runningTasks", ImmutableSet.of("first", "second"), runningTasks);
}

Example 2 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class RemoteTaskRunnerTest method testSuccessfulTaskOnBlacklistedWorker.

@Test
public void testSuccessfulTaskOnBlacklistedWorker() throws Exception {
    makeWorker();
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(100);
    makeRemoteTaskRunner(rtrConfig);
    TestRealtimeTask task1 = new TestRealtimeTask("realtime1", new TaskResource("realtime1", 1), "foo", TaskStatus.success("realtime1"), jsonMapper);
    TestRealtimeTask task2 = new TestRealtimeTask("realtime2", new TaskResource("realtime2", 1), "foo", TaskStatus.success("realtime2"), jsonMapper);
    TestRealtimeTask task3 = new TestRealtimeTask("realtime3", new TaskResource("realtime3", 1), "foo", TaskStatus.success("realtime3"), jsonMapper);
    Future<TaskStatus> taskFuture1 = remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    mockWorkerCompleteFailedTask(task1);
    Assert.assertTrue(taskFuture1.get().isFailure());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture2 = remoteTaskRunner.run(task2);
    Assert.assertTrue(taskAnnounced(task2.getId()));
    mockWorkerRunningTask(task2);
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture3 = remoteTaskRunner.run(task3);
    Assert.assertTrue(taskAnnounced(task3.getId()));
    mockWorkerRunningTask(task3);
    mockWorkerCompleteFailedTask(task3);
    Assert.assertTrue(taskFuture3.get().isFailure());
    Assert.assertEquals(1, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertEquals(3, remoteTaskRunner.getBlacklistedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    mockWorkerCompleteSuccessfulTask(task2);
    Assert.assertTrue(taskFuture2.get().isSuccess());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
}

Example 3 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testSuccessfulMinWorkersProvision.

@Test
public void testSuccessfulMinWorkersProvision() {
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(3).times(2);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(5);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.anyObject())).andReturn(new ArrayList<String>());
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    // No pending tasks
    EasyMock.expect(runner.getPendingTaskPayloads()).andReturn(new ArrayList<>());
    // 1 node already running, only provision 2 more.
    EasyMock.expect(runner.getWorkers()).andReturn(Collections.singletonList(new TestZkWorker(testTask).toImmutable()));
    EasyMock.expect(runner.getConfig()).andReturn(new RemoteTaskRunnerConfig());
    EasyMock.expect(autoScaler.provision()).andReturn(new AutoScalingData(Collections.singletonList("aNode"))).times(2);
    EasyMock.replay(runner, autoScaler);
    Provisioner provisioner = strategy.makeProvisioner(runner);
    boolean provisionedSomething = provisioner.doProvision();
    Assert.assertTrue(provisionedSomething);
    Assert.assertTrue(provisioner.getStats().toList().size() == 2);
    for (ScalingStats.ScalingEvent event : provisioner.getStats().toList()) {
        Assert.assertTrue(event.getEvent() == ScalingStats.EVENT.PROVISION);
    }
}

Example 4 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class EqualDistributionWithAffinityWorkerSelectStrategyTest method testIsolation.

@Test
public void testIsolation() {
    EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost")), false));
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost", new ImmutableWorkerInfo(new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null));
    Assert.assertNull(worker);
}

Example 5 with RemoteTaskRunnerConfig

use of org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig in project druid by druid-io.

the class EqualDistributionWithAffinityWorkerSelectStrategyTest method testFindWorkerForTask.

@Test
public void testFindWorkerForTask() {
    EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost1", "localhost2", "localhost3")), false));
    NoopTask noopTask = new NoopTask(null, null, null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    };
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost0", new ImmutableWorkerInfo(new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost1", new ImmutableWorkerInfo(new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost2", new ImmutableWorkerInfo(new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()), "localhost3", new ImmutableWorkerInfo(new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), noopTask);
    Assert.assertEquals("localhost1", worker.getWorker().getHost());
}