Examples with WorkerHistoryItem org.apache.druid.indexing.worker.WorkerHistoryItem used on opensource projects

Example 1 with WorkerHistoryItem

use of org.apache.druid.indexing.worker.WorkerHistoryItem in project druid by druid-io.

the class WorkerHolder method createSyncListener.

public ChangeRequestHttpSyncer.Listener<WorkerHistoryItem> createSyncListener() {
    return new ChangeRequestHttpSyncer.Listener<WorkerHistoryItem>() {

        @Override
        public void fullSync(List<WorkerHistoryItem> changes) {
            ConcurrentMap<String, TaskAnnouncement> newSnapshot = new ConcurrentHashMap<>();
            List<TaskAnnouncement> delta = new ArrayList<>();
            boolean isWorkerDisabled = disabled.get();
            for (WorkerHistoryItem change : changes) {
                if (change instanceof WorkerHistoryItem.TaskUpdate) {
                    TaskAnnouncement announcement = ((WorkerHistoryItem.TaskUpdate) change).getTaskAnnouncement();
                    newSnapshot.put(announcement.getTaskId(), announcement);
                    delta.add(announcement);
                } else if (change instanceof WorkerHistoryItem.Metadata) {
                    isWorkerDisabled = ((WorkerHistoryItem.Metadata) change).isDisabled();
                } else {
                    log.makeAlert("Got unknown sync update[%s] from worker[%s]. Ignored.", change.getClass().getName(), worker.getHost()).emit();
                }
            }
            for (TaskAnnouncement announcement : tasksSnapshotRef.get().values()) {
                if (!newSnapshot.containsKey(announcement.getTaskId()) && !announcement.getTaskStatus().isComplete()) {
                    log.warn("task[%s] in state[%s] suddenly disappeared on worker[%s]. failing it.", announcement.getTaskId(), announcement.getStatus(), worker.getHost());
                    delta.add(TaskAnnouncement.create(announcement.getTaskId(), announcement.getTaskType(), announcement.getTaskResource(), TaskStatus.failure(announcement.getTaskId(), "This task disappeared on the worker where it was assigned. " + "See overlord logs for more details."), announcement.getTaskLocation(), announcement.getTaskDataSource()));
                }
            }
            tasksSnapshotRef.set(newSnapshot);
            notifyListener(delta, isWorkerDisabled);
        }

        @Override
        public void deltaSync(List<WorkerHistoryItem> changes) {
            List<TaskAnnouncement> delta = new ArrayList<>();
            boolean isWorkerDisabled = disabled.get();
            for (WorkerHistoryItem change : changes) {
                if (change instanceof WorkerHistoryItem.TaskUpdate) {
                    TaskAnnouncement announcement = ((WorkerHistoryItem.TaskUpdate) change).getTaskAnnouncement();
                    tasksSnapshotRef.get().put(announcement.getTaskId(), announcement);
                    delta.add(announcement);
                } else if (change instanceof WorkerHistoryItem.TaskRemoval) {
                    String taskId = ((WorkerHistoryItem.TaskRemoval) change).getTaskId();
                    TaskAnnouncement announcement = tasksSnapshotRef.get().remove(taskId);
                    if (announcement != null && !announcement.getTaskStatus().isComplete()) {
                        log.warn("task[%s] in state[%s] suddenly disappeared on worker[%s]. failing it.", announcement.getTaskId(), announcement.getStatus(), worker.getHost());
                        delta.add(TaskAnnouncement.create(announcement.getTaskId(), announcement.getTaskType(), announcement.getTaskResource(), TaskStatus.failure(announcement.getTaskId(), "This task disappeared on the worker where it was assigned. " + "See overlord logs for more details."), announcement.getTaskLocation(), announcement.getTaskDataSource()));
                    }
                } else if (change instanceof WorkerHistoryItem.Metadata) {
                    isWorkerDisabled = ((WorkerHistoryItem.Metadata) change).isDisabled();
                } else {
                    log.makeAlert("Got unknown sync update[%s] from worker[%s]. Ignored.", change.getClass().getName(), worker.getHost()).emit();
                }
            }
            notifyListener(delta, isWorkerDisabled);
        }

        private void notifyListener(List<TaskAnnouncement> announcements, boolean isWorkerDisabled) {
            for (TaskAnnouncement announcement : announcements) {
                try {
                    listener.taskAddedOrUpdated(announcement, WorkerHolder.this);
                } catch (Exception ex) {
                    log.error(ex, "Unknown exception while updating task[%s] state from worker[%s].", announcement.getTaskId(), worker.getHost());
                }
            }
            if (isWorkerDisabled != disabled.get()) {
                disabled.set(isWorkerDisabled);
                log.info("Worker[%s] disabled set to [%s].", worker.getHost(), isWorkerDisabled);
            }
        }
    };
}

Example 2 with WorkerHistoryItem

use of org.apache.druid.indexing.worker.WorkerHistoryItem in project druid by druid-io.

the class WorkerHolderTest method testSyncListener.

@Test
public void testSyncListener() {
    List<TaskAnnouncement> updates = new ArrayList<>();
    Task task0 = NoopTask.create("task0", 0);
    Task task1 = NoopTask.create("task1", 0);
    Task task2 = NoopTask.create("task2", 0);
    Task task3 = NoopTask.create("task3", 0);
    WorkerHolder workerHolder = new WorkerHolder(TestHelper.makeJsonMapper(), EasyMock.createNiceMock(HttpClient.class), new HttpRemoteTaskRunnerConfig(), EasyMock.createNiceMock(ScheduledExecutorService.class), (taskAnnouncement, holder) -> updates.add(taskAnnouncement), new Worker("http", "localhost", "127.0.0.1", 5, "v0", WorkerConfig.DEFAULT_CATEGORY), ImmutableList.of(TaskAnnouncement.create(task0, TaskStatus.running(task0.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown())));
    ChangeRequestHttpSyncer.Listener<WorkerHistoryItem> syncListener = workerHolder.createSyncListener();
    Assert.assertTrue(workerHolder.disabled.get());
    syncListener.fullSync(ImmutableList.of(new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("w1", 1, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("w1", 2, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 2, -1)))));
    Assert.assertFalse(workerHolder.disabled.get());
    Assert.assertEquals(4, updates.size());
    Assert.assertEquals(task1.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isSuccess());
    Assert.assertEquals(task2.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    Assert.assertEquals(task3.getId(), updates.get(2).getTaskId());
    Assert.assertTrue(updates.get(2).getTaskStatus().isRunnable());
    Assert.assertEquals(task0.getId(), updates.get(3).getTaskId());
    Assert.assertTrue(updates.get(3).getTaskStatus().isFailure());
    Assert.assertNotNull(updates.get(3).getTaskStatus().getErrorMsg());
    Assert.assertTrue(updates.get(3).getTaskStatus().getErrorMsg().startsWith("This task disappeared on the worker where it was assigned"));
    updates.clear();
    syncListener.deltaSync(ImmutableList.of(new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskRemoval(task1.getId()), new WorkerHistoryItem.Metadata(true), new WorkerHistoryItem.TaskRemoval(task2.getId()), new WorkerHistoryItem.Metadata(false), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 3, -1)))));
    Assert.assertFalse(workerHolder.disabled.get());
    Assert.assertEquals(2, updates.size());
    Assert.assertEquals(task2.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isFailure());
    Assert.assertNotNull(updates.get(0).getTaskStatus().getErrorMsg());
    Assert.assertTrue(updates.get(0).getTaskStatus().getErrorMsg().startsWith("This task disappeared on the worker where it was assigned"));
    Assert.assertEquals(task3.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    updates.clear();
    syncListener.fullSync(ImmutableList.of(new WorkerHistoryItem.Metadata(true), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("w1", 1, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("w1", 2, -1))), new WorkerHistoryItem.TaskUpdate(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("w1", 2, -1)))));
    Assert.assertTrue(workerHolder.disabled.get());
    Assert.assertEquals(3, updates.size());
    Assert.assertEquals(task1.getId(), updates.get(0).getTaskId());
    Assert.assertTrue(updates.get(0).getTaskStatus().isSuccess());
    Assert.assertEquals(task2.getId(), updates.get(1).getTaskId());
    Assert.assertTrue(updates.get(1).getTaskStatus().isRunnable());
    Assert.assertEquals(task3.getId(), updates.get(2).getTaskId());
    Assert.assertTrue(updates.get(2).getTaskStatus().isRunnable());
    updates.clear();
}