Examples with TestRealtimeTask org.apache.druid.indexing.common.TestRealtimeTask used on opensource projects

Example 1 with TestRealtimeTask

use of org.apache.druid.indexing.common.TestRealtimeTask in project druid by druid-io.

the class RemoteTaskRunnerTest method testRunSameAvailabilityGroup.

@Test
public void testRunSameAvailabilityGroup() throws Exception {
    doSetup();
    TestRealtimeTask task1 = new TestRealtimeTask("rt1", new TaskResource("rt1", 1), "foo", TaskStatus.running("rt1"), jsonMapper);
    remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    TestRealtimeTask task2 = new TestRealtimeTask("rt2", new TaskResource("rt1", 1), "foo", TaskStatus.running("rt2"), jsonMapper);
    remoteTaskRunner.run(task2);
    TestRealtimeTask task3 = new TestRealtimeTask("rt3", new TaskResource("rt2", 1), "foo", TaskStatus.running("rt3"), jsonMapper);
    remoteTaskRunner.run(task3);
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getRunningTasks().size() == 2;
        }
    }));
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getPendingTasks().size() == 1;
        }
    }));
    Assert.assertTrue(remoteTaskRunner.getPendingTasks().iterator().next().getTaskId().equals("rt2"));
}

Example 2 with TestRealtimeTask

use of org.apache.druid.indexing.common.TestRealtimeTask in project druid by druid-io.

the class RemoteTaskRunnerTest method testRunWithTaskComplete.

@Test
public void testRunWithTaskComplete() throws Exception {
    doSetup();
    TestRealtimeTask task1 = new TestRealtimeTask("testTask", new TaskResource("testTask", 2), "foo", TaskStatus.success("testTask"), jsonMapper);
    remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    mockWorkerCompleteSuccessfulTask(task1);
    Assert.assertEquals(TaskState.SUCCESS, remoteTaskRunner.run(task1).get().getStatusCode());
}

Example 3 with TestRealtimeTask

use of org.apache.druid.indexing.common.TestRealtimeTask in project druid by druid-io.

the class RemoteTaskRunnerTest method testBootstrap.

@Test
public void testBootstrap() throws Exception {
    makeWorker();
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(100);
    makeRemoteTaskRunner(rtrConfig);
    TestRealtimeTask task1 = new TestRealtimeTask("first", new TaskResource("first", 1), "foo", TaskStatus.running("first"), jsonMapper);
    remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    TestRealtimeTask task = new TestRealtimeTask("second", new TaskResource("task", 2), "foo", TaskStatus.running("task"), jsonMapper);
    remoteTaskRunner.run(task);
    TestRealtimeTask task2 = new TestRealtimeTask("second", new TaskResource("second", 2), "foo", TaskStatus.running("second"), jsonMapper);
    remoteTaskRunner.run(task2);
    Assert.assertTrue(taskAnnounced(task2.getId()));
    mockWorkerRunningTask(task2);
    final Set<String> runningTasks = Sets.newHashSet(Iterables.transform(remoteTaskRunner.getRunningTasks(), new Function<RemoteTaskRunnerWorkItem, String>() {

        @Override
        public String apply(RemoteTaskRunnerWorkItem input) {
            return input.getTaskId();
        }
    }));
    Assert.assertEquals("runningTasks", ImmutableSet.of("first", "second"), runningTasks);
}

Example 4 with TestRealtimeTask

use of org.apache.druid.indexing.common.TestRealtimeTask in project druid by druid-io.

the class RemoteTaskRunnerTest method testSuccessfulTaskOnBlacklistedWorker.

@Test
public void testSuccessfulTaskOnBlacklistedWorker() throws Exception {
    makeWorker();
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(100);
    makeRemoteTaskRunner(rtrConfig);
    TestRealtimeTask task1 = new TestRealtimeTask("realtime1", new TaskResource("realtime1", 1), "foo", TaskStatus.success("realtime1"), jsonMapper);
    TestRealtimeTask task2 = new TestRealtimeTask("realtime2", new TaskResource("realtime2", 1), "foo", TaskStatus.success("realtime2"), jsonMapper);
    TestRealtimeTask task3 = new TestRealtimeTask("realtime3", new TaskResource("realtime3", 1), "foo", TaskStatus.success("realtime3"), jsonMapper);
    Future<TaskStatus> taskFuture1 = remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    mockWorkerCompleteFailedTask(task1);
    Assert.assertTrue(taskFuture1.get().isFailure());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture2 = remoteTaskRunner.run(task2);
    Assert.assertTrue(taskAnnounced(task2.getId()));
    mockWorkerRunningTask(task2);
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Future<TaskStatus> taskFuture3 = remoteTaskRunner.run(task3);
    Assert.assertTrue(taskAnnounced(task3.getId()));
    mockWorkerRunningTask(task3);
    mockWorkerCompleteFailedTask(task3);
    Assert.assertTrue(taskFuture3.get().isFailure());
    Assert.assertEquals(1, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertEquals(3, remoteTaskRunner.getBlacklistedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    mockWorkerCompleteSuccessfulTask(task2);
    Assert.assertTrue(taskFuture2.get().isSuccess());
    Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
    Assert.assertFalse(remoteTaskRunner.getBlacklistedTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
}

Example 5 with TestRealtimeTask

use of org.apache.druid.indexing.common.TestRealtimeTask in project druid by druid-io.

the class RemoteTaskRunnerTest method testBlacklistZKWorkers25Percent.

/**
 * With 2 workers and maxPercentageBlacklistWorkers(25), neither worker should ever be blacklisted even after
 * exceeding maxRetriesBeforeBlacklist.
 */
@Test
public void testBlacklistZKWorkers25Percent() throws Exception {
    rtrTestUtils.makeWorker("worker", 10);
    rtrTestUtils.makeWorker("worker2", 10);
    RemoteTaskRunnerConfig rtrConfig = new TestRemoteTaskRunnerConfig(TIMEOUT_PERIOD);
    rtrConfig.setMaxPercentageBlacklistWorkers(25);
    makeRemoteTaskRunner(rtrConfig);
    String firstWorker = null;
    String secondWorker = null;
    for (int i = 1; i < 13; i++) {
        String taskId = StringUtils.format("rt-%d", i);
        TestRealtimeTask task = new TestRealtimeTask(taskId, new TaskResource(taskId, 1), "foo", TaskStatus.success(taskId), jsonMapper);
        Future<TaskStatus> taskFuture = remoteTaskRunner.run(task);
        if (i == 1) {
            if (rtrTestUtils.taskAnnounced("worker2", task.getId())) {
                firstWorker = "worker2";
                secondWorker = "worker";
            } else {
                firstWorker = "worker";
                secondWorker = "worker2";
            }
        }
        final String expectedWorker = i % 2 == 0 ? secondWorker : firstWorker;
        Assert.assertTrue(rtrTestUtils.taskAnnounced(expectedWorker, task.getId()));
        rtrTestUtils.mockWorkerRunningTask(expectedWorker, task);
        rtrTestUtils.mockWorkerCompleteFailedTask(expectedWorker, task);
        Assert.assertTrue(taskFuture.get().isFailure());
        Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size());
        Assert.assertEquals(((i + 1) / 2), remoteTaskRunner.findWorkerRunningTask(task.getId()).getContinuouslyFailedTasksCount());
    }
}