Examples with TaskAnnouncement org.apache.druid.indexing.worker.TaskAnnouncement used on opensource projects

Example 6 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class HttpRemoteTaskRunner method addWorker.

private void addWorker(final Worker worker) {
    synchronized (workers) {
        log.info("Worker[%s] reportin' for duty!", worker.getHost());
        cancelWorkerCleanup(worker.getHost());
        WorkerHolder holder = workers.get(worker.getHost());
        if (holder == null) {
            List<TaskAnnouncement> expectedAnnouncements = new ArrayList<>();
            synchronized (statusLock) {
                // manages the task syncing with that worker.
                for (Map.Entry<String, HttpRemoteTaskRunnerWorkItem> e : tasks.entrySet()) {
                    if (e.getValue().getState() == HttpRemoteTaskRunnerWorkItem.State.RUNNING) {
                        Worker w = e.getValue().getWorker();
                        if (w != null && w.getHost().equals(worker.getHost())) {
                            expectedAnnouncements.add(TaskAnnouncement.create(e.getValue().getTask(), TaskStatus.running(e.getKey()), e.getValue().getLocation()));
                        }
                    }
                }
            }
            holder = createWorkerHolder(smileMapper, httpClient, config, workersSyncExec, this::taskAddedOrUpdated, worker, expectedAnnouncements);
            holder.start();
            workers.put(worker.getHost(), holder);
        } else {
            log.info("Worker[%s] already exists.", worker.getHost());
        }
    }
    synchronized (statusLock) {
        statusLock.notifyAll();
    }
}

Example 7 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class WorkerHolderTest method testSyncListener.

@Test
public void testSyncListener() {
    List<TaskAnnouncement> updates = new ArrayList<>();
    Task task0 = NoopTask.create("task0", 0);
    Task task1 = NoopTask.create("task1", 0);
    Task task2 = NoopTask.create("task2", 0);
    Task task3 = NoopTask.create("task3", 0);
    WorkerHolder workerHolder = new WorkerHolder(TestHelper.makeJsonMapper(), EasyMock.createNiceMock(HttpClient.class), new HttpRemoteTaskRunnerConfig(), EasyMock.createNiceMock(ScheduledExecutorService.class), (taskAnnouncement, holder) -> updates.add(taskAnnouncement), new Worker("http", "localhost", "127.0.0.1", 5, "v0", WorkerConfig.DEFAULT_CATEGORY), ImmutableList.of(TaskAnnouncement.create(task0, TaskStatus.running(task0.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown())));
    ChangeRequestHttpSyncer.Listener<WorkerHistoryItem> syncListener = workerHolder.createSyncListener();
    Assert.assertTrue(workerHolder.disabled.get());
    syncListener.fullSync(ImmutableList.of(new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("w1", 1, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("w1", 2, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 2, -1)))));
    Assert.assertFalse(workerHolder.disabled.get());
    Assert.assertEquals(4, updates.size());
    Assert.assertEquals(task1.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isSuccess());
    Assert.assertEquals(task2.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    Assert.assertEquals(task3.getId(), updates.get(2).getTaskId());
    Assert.assertTrue(updates.get(2).getTaskStatus().isRunnable());
    Assert.assertEquals(task0.getId(), updates.get(3).getTaskId());
    Assert.assertTrue(updates.get(3).getTaskStatus().isFailure());
    Assert.assertNotNull(updates.get(3).getTaskStatus().getErrorMsg());
    Assert.assertTrue(updates.get(3).getTaskStatus().getErrorMsg().startsWith("This task disappeared on the worker where it was assigned"));
    updates.clear();
    syncListener.deltaSync(ImmutableList.of(new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskRemoval(task1.getId()), new WorkerHistoryItem.Metadata(true), new WorkerHistoryItem.TaskRemoval(task2.getId()), new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 3, -1)))));
    Assert.assertFalse(workerHolder.disabled.get());
    Assert.assertEquals(2, updates.size());
    Assert.assertEquals(task2.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isFailure());
    Assert.assertNotNull(updates.get(0).getTaskStatus().getErrorMsg());
    Assert.assertTrue(updates.get(0).getTaskStatus().getErrorMsg().startsWith("This task disappeared on the worker where it was assigned"));
    Assert.assertEquals(task3.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    updates.clear();
    syncListener.fullSync(ImmutableList.of(new WorkerHistoryItem.Metadata(true), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("w1", 1, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("w1", 2, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 2, -1)))));
    Assert.assertTrue(workerHolder.disabled.get());
    Assert.assertEquals(3, updates.size());
    Assert.assertEquals(task1.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isSuccess());
    Assert.assertEquals(task2.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    Assert.assertEquals(task3.getId(), updates.get(2).getTaskId());
    Assert.assertTrue(updates.get(2).getTaskStatus().isRunnable());
    updates.clear();
}