Examples with TaskStatus org.apache.druid.indexer.TaskStatus used on opensource projects

Example 51 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testTaskAddedOrUpdated1.

/*
   * Task goes PENDING -> RUNNING -> SUCCESS and few more useless notifications in between.
   */
@Test
public void testTaskAddedOrUpdated1() throws Exception {
    Task task = NoopTask.create("task");
    List<Object> listenerNotificationsAccumulator = new ArrayList<>();
    HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(EasyMock.createStrictMock(TaskStorage.class), listenerNotificationsAccumulator);
    WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
    workerHolder.setLastCompletedTaskTime(EasyMock.anyObject());
    workerHolder.resetContinuouslyFailedTasksCount();
    EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0);
    EasyMock.replay(workerHolder);
    Future<TaskStatus> future = taskRunner.run(task);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
    // RUNNING notification from worker
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1000, 1001)), workerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    // Another RUNNING notification from worker, notifying change in location
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    // Redundant RUNNING notification from worker, ignored
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    // Another "rogue-worker" reports running it, and gets asked to shutdown the task
    WorkerHolder rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
    rogueWorkerHolder.shutdownTask(task.getId());
    EasyMock.replay(rogueWorkerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    EasyMock.verify(rogueWorkerHolder);
    // "rogue-worker" reports FAILURE for the task, ignored
    rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
    EasyMock.replay(rogueWorkerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.failure(task.getId(), "Dummy task status failure err message"), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    EasyMock.verify(rogueWorkerHolder);
    // workers sends SUCCESS notification, task is marked SUCCESS now.
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.success(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
    Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
    // "rogue-worker" reports running it, and gets asked to shutdown the task
    rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
    rogueWorkerHolder.shutdownTask(task.getId());
    EasyMock.replay(rogueWorkerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
    EasyMock.verify(rogueWorkerHolder);
    // "rogue-worker" reports FAILURE for the tasks, ignored
    rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(rogueWorkerHolder.getWorker()).andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes();
    EasyMock.replay(rogueWorkerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.failure(task.getId(), "Dummy task status failure for testing"), TaskLocation.create("rogue-worker", 1, 2)), rogueWorkerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
    EasyMock.verify(rogueWorkerHolder);
    Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
    EasyMock.verify(workerHolder);
    Assert.assertEquals(listenerNotificationsAccumulator, ImmutableList.of(ImmutableList.of(task.getId(), TaskLocation.create("worker", 1000, 1001)), ImmutableList.of(task.getId(), TaskLocation.create("worker", 1, 2)), ImmutableList.of(task.getId(), TaskStatus.success(task.getId()))));
}

Example 52 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testTimeoutInAssigningTasks.

@Test
public void testTimeoutInAssigningTasks() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 1;
        }

        @Override
        public Period getTaskAssignmentTimeout() {
            return new Period("PT1S");
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            return new WorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of()) {

                @Override
                public void start() {
                    disabled.set(false);
                }

                @Override
                public void stop() {
                }

                @Override
                public boolean isInitialized() {
                    return true;
                }

                @Override
                public void waitForInitialization() {
                }

                @Override
                public boolean assignTask(Task task) {
                    // Always returns true
                    return true;
                }

                @Override
                public void shutdownTask(String taskId) {
                }
            };
        }
    };
    taskRunner.start();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
    Future<TaskStatus> future = taskRunner.run(NoopTask.create("task-id", 0));
    Assert.assertTrue(future.get().isFailure());
    Assert.assertNotNull(future.get().getErrorMsg());
    Assert.assertTrue(future.get().getErrorMsg().startsWith("The worker that this task is assigned did not start it in timeout"));
}

Example 53 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testTaskAddedOrUpdated2.

/*
   * Task goes from PENDING -> SUCCESS . Happens when TaskRunner is given task but a worker reported it being already
   * completed with SUCCESS.
   */
@Test
public void testTaskAddedOrUpdated2() throws Exception {
    Task task = NoopTask.create("task");
    List<Object> listenerNotificationsAccumulator = new ArrayList<>();
    HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(EasyMock.createStrictMock(TaskStorage.class), listenerNotificationsAccumulator);
    Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY);
    WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes();
    workerHolder.setLastCompletedTaskTime(EasyMock.anyObject());
    workerHolder.resetContinuouslyFailedTasksCount();
    EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0);
    EasyMock.replay(workerHolder);
    Future<TaskStatus> future = taskRunner.run(task);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task, TaskStatus.success(task.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
    Assert.assertEquals(task.getId(), Iterables.getOnlyElement(taskRunner.getCompletedTasks()).getTaskId());
    Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode());
    EasyMock.verify(workerHolder);
    Assert.assertEquals(listenerNotificationsAccumulator, ImmutableList.of(ImmutableList.of(task.getId(), TaskLocation.create("worker", 1, 2)), ImmutableList.of(task.getId(), TaskStatus.success(task.getId()))));
}

Example 54 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testExceptionThrownInAssigningTasks.

@Test
public void testExceptionThrownInAssigningTasks() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 1;
        }

        @Override
        public Period getTaskAssignmentTimeout() {
            return new Period("PT1S");
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            return new WorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of()) {

                @Override
                public void start() {
                    disabled.set(false);
                }

                @Override
                public void stop() {
                }

                @Override
                public boolean isInitialized() {
                    return true;
                }

                @Override
                public void waitForInitialization() {
                }

                @Override
                public boolean assignTask(Task task) {
                    throw new RuntimeException("Assign failure test");
                }

                @Override
                public void shutdownTask(String taskId) {
                }
            };
        }
    };
    taskRunner.start();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
    Future<TaskStatus> future = taskRunner.run(NoopTask.create("task-id", 0));
    Assert.assertTrue(future.get().isFailure());
    Assert.assertNotNull(future.get().getErrorMsg());
    Assert.assertTrue(StringUtils.format("Actual message is: %s", future.get().getErrorMsg()), future.get().getErrorMsg().startsWith("Failed to assign this task"));
}

Example 55 with TaskStatus

use of org.apache.druid.indexer.TaskStatus in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testFreshStart.

/*
  Simulates startup of Overlord and Workers being discovered with no previously known tasks. Fresh tasks are given
  and expected to be completed.
   */
@Test(timeout = 60_000L)
public void testFreshStart() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    HttpRemoteTaskRunner taskRunner = newHttpTaskRunnerInstance(druidNodeDiscoveryProvider, new NoopProvisioningStrategy<>());
    taskRunner.start();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1, druidNode2));
    int numTasks = 8;
    List<Future<TaskStatus>> futures = new ArrayList<>();
    for (int i = 0; i < numTasks; i++) {
        futures.add(taskRunner.run(NoopTask.create("task-id-" + i, 0)));
    }
    for (Future<TaskStatus> future : futures) {
        Assert.assertTrue(future.get().isSuccess());
    }
    Assert.assertEquals(numTasks, taskRunner.getKnownTasks().size());
    Assert.assertEquals(numTasks, taskRunner.getCompletedTasks().size());
}