use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.
the class HttpRemoteTaskRunnerTest method testTaskAddedOrUpdated1.
/*
* Task goes PENDING -> RUNNING -> SUCCESS and few more useless notifications in between.
*/
@Test
public void testTaskAddedOrUpdated1() throws Exception {
Task task = NoopTask.create("task");
List<Object> listenerNotificationsAccumulator = new ArrayList<>();
HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(EasyMock.createStrictMock(TaskStorage.class), listenerNotificationsAccumulator);
WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
workerHolder.setLastCompletedTaskTime(EasyMock.anyObject());
workerHolder.resetContinuouslyFailedTasksCount();
EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0);
EasyMock.replay(workerHolder);
Future<TaskStatus> future = taskRunner.run(task);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
// RUNNING notification from worker
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1000, 1001)), workerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
// Another RUNNING notification from worker, notifying change in location
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
// Redundant RUNNING notification from worker, ignored
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
// Another "rogue-worker" reports running it, and gets asked to shutdown the task
WorkerHolder rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
rogueWorkerHolder.shutdownTask(task.getId());
EasyMock.replay(rogueWorkerHolder);
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
EasyMock.verify(rogueWorkerHolder);
// "rogue-worker" reports FAILURE for the task, ignored
rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
EasyMock.replay(rogueWorkerHolder);
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.failure(task.getId(), "Dummy task status failure err message"), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
EasyMock.verify(rogueWorkerHolder);
// workers sends SUCCESS notification, task is marked SUCCESS now.
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.success(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
// "rogue-worker" reports running it, and gets asked to shutdown the task
rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
rogueWorkerHolder.shutdownTask(task.getId());
EasyMock.replay(rogueWorkerHolder);
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
EasyMock.verify(rogueWorkerHolder);
// "rogue-worker" reports FAILURE for the tasks, ignored
rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
EasyMock.replay(rogueWorkerHolder);
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.failure(task.getId(), "Dummy task status failure for testing"), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
EasyMock.verify(rogueWorkerHolder);
Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
EasyMock.verify(workerHolder);
Assert.assertEquals(listenerNotificationsAccumulator, ImmutableList.of(ImmutableList.of(task.getId(), TaskLocation.create("worker", 1000, 1001)), ImmutableList.of(task.getId(), TaskLocation.create("worker", 1, 2)), ImmutableList.of(task.getId(), TaskStatus.success(task.getId()))));
}
use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.
the class HttpRemoteTaskRunnerTest method testTimeoutInAssigningTasks.
@Test
public void testTimeoutInAssigningTasks() throws Exception {
TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
EasyMock.replay(druidNodeDiscoveryProvider);
HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {
@Override
public int getPendingTasksRunnerNumThreads() {
return 1;
}
@Override
public Period getTaskAssignmentTimeout() {
return new Period("PT1S");
}
}, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {
@Override
protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
return new WorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of()) {
@Override
public void start() {
disabled.set(false);
}
@Override
public void stop() {
}
@Override
public boolean isInitialized() {
return true;
}
@Override
public void waitForInitialization() {
}
@Override
public boolean assignTask(Task task) {
// Always returns true
return true;
}
@Override
public void shutdownTask(String taskId) {
}
};
}
};
taskRunner.start();
DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
Future<TaskStatus> future = taskRunner.run(NoopTask.create("task-id", 0));
Assert.assertTrue(future.get().isFailure());
Assert.assertNotNull(future.get().getErrorMsg());
Assert.assertTrue(future.get().getErrorMsg().startsWith("The worker that this task is assigned did not start it in timeout"));
}
use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.
the class HttpRemoteTaskRunnerTest method testTaskRunnerRestart.
/*
Simulates restart of the Overlord where taskRunner, on start, discovers workers with prexisting tasks.
*/
@Test(timeout = 60_000L)
public void testTaskRunnerRestart() throws Exception {
TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
EasyMock.replay(druidNodeDiscoveryProvider);
ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
Task task1 = NoopTask.create("task-id-1", 0);
Task task2 = NoopTask.create("task-id-2", 0);
Task task3 = NoopTask.create("task-id-3", 0);
Task task4 = NoopTask.create("task-id-4", 0);
Task task5 = NoopTask.create("task-id-5", 0);
TaskStorage taskStorageMock = EasyMock.createStrictMock(TaskStorage.class);
EasyMock.expect(taskStorageMock.getStatus(task1.getId())).andReturn(Optional.absent());
EasyMock.expect(taskStorageMock.getStatus(task2.getId())).andReturn(Optional.absent()).times(2);
EasyMock.expect(taskStorageMock.getStatus(task3.getId())).andReturn(Optional.of(TaskStatus.running(task3.getId())));
EasyMock.expect(taskStorageMock.getStatus(task4.getId())).andReturn(Optional.of(TaskStatus.running(task4.getId())));
EasyMock.expect(taskStorageMock.getStatus(task5.getId())).andReturn(Optional.of(TaskStatus.success(task5.getId())));
EasyMock.replay(taskStorageMock);
HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {
@Override
public int getPendingTasksRunnerNumThreads() {
return 3;
}
}, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, taskStorageMock, EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {
@Override
protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
if (workerHolders.containsKey(worker.getHost())) {
return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
} else {
throw new ISE("No WorkerHolder for [%s].", worker.getHost());
}
}
};
taskRunner.start();
DiscoveryDruidNode druidNode = new DiscoveryDruidNode(new DruidNode("service", "host", false, 1234, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
AtomicInteger ticks = new AtomicInteger();
Set<String> taskShutdowns = new HashSet<>();
workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task2, TaskStatus.success(task2.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task3, TaskStatus.success(task3.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task4, TaskStatus.running(task4.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task5, TaskStatus.running(task5.getId()), TaskLocation.create("host", 1234, 1235))), ImmutableMap.of(), ticks, taskShutdowns));
druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
while (ticks.get() < 1) {
Thread.sleep(100);
}
EasyMock.verify(taskStorageMock);
Assert.assertEquals(ImmutableSet.of(task2.getId(), task5.getId()), taskShutdowns);
Assert.assertTrue(taskRunner.getPendingTasks().isEmpty());
TaskRunnerWorkItem item = Iterables.getOnlyElement(taskRunner.getRunningTasks());
Assert.assertEquals(task4.getId(), item.getTaskId());
Assert.assertTrue(taskRunner.run(task3).get().isSuccess());
Assert.assertEquals(2, taskRunner.getKnownTasks().size());
}
use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.
the class HttpRemoteTaskRunnerTest method testTaskAddedOrUpdated2.
/*
* Task goes from PENDING -> SUCCESS . Happens when TaskRunner is given task but a worker reported it being already
* completed with SUCCESS.
*/
@Test
public void testTaskAddedOrUpdated2() throws Exception {
Task task = NoopTask.create("task");
List<Object> listenerNotificationsAccumulator = new ArrayList<>();
HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(EasyMock.createStrictMock(TaskStorage.class), listenerNotificationsAccumulator);
Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY);
WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class);
EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes();
workerHolder.setLastCompletedTaskTime(EasyMock.anyObject());
workerHolder.resetContinuouslyFailedTasksCount();
EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0);
EasyMock.replay(workerHolder);
Future<TaskStatus> future = taskRunner.run(task);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.success(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
EasyMock.verify(workerHolder);
Assert.assertEquals(listenerNotificationsAccumulator, ImmutableList.of(ImmutableList.of(task.getId(), TaskLocation.create("worker", 1, 2)), ImmutableList.of(task.getId(), TaskStatus.success(task.getId()))));
}
use of org.apache.druid.indexing.common.task.Task in project druid by druid-io.
the class HttpRemoteTaskRunnerTest method testExceptionThrownInAssigningTasks.
@Test
public void testExceptionThrownInAssigningTasks() throws Exception {
TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
EasyMock.replay(druidNodeDiscoveryProvider);
HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {
@Override
public int getPendingTasksRunnerNumThreads() {
return 1;
}
@Override
public Period getTaskAssignmentTimeout() {
return new Period("PT1S");
}
}, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {
@Override
protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
return new WorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of()) {
@Override
public void start() {
disabled.set(false);
}
@Override
public void stop() {
}
@Override
public boolean isInitialized() {
return true;
}
@Override
public void waitForInitialization() {
}
@Override
public boolean assignTask(Task task) {
throw new RuntimeException("Assign failure test");
}
@Override
public void shutdownTask(String taskId) {
}
};
}
};
taskRunner.start();
DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
Future<TaskStatus> future = taskRunner.run(NoopTask.create("task-id", 0));
Assert.assertTrue(future.get().isFailure());
Assert.assertNotNull(future.get().getErrorMsg());
Assert.assertTrue(StringUtils.format("Actual message is: %s", future.get().getErrorMsg()), future.get().getErrorMsg().startsWith("Failed to assign this task"));
}
Aggregations