Examples with TaskAnnouncement org.apache.druid.indexing.worker.TaskAnnouncement used on opensource projects

Example 1 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class WorkerHolder method createSyncListener.

public ChangeRequestHttpSyncer.Listener<WorkerHistoryItem> createSyncListener() {
    return new ChangeRequestHttpSyncer.Listener<WorkerHistoryItem>() {

        @Override
        public void fullSync(List<WorkerHistoryItem> changes) {
            ConcurrentMap<String, TaskAnnouncement> newSnapshot = new ConcurrentHashMap<>();
            List<TaskAnnouncement> delta = new ArrayList<>();
            boolean isWorkerDisabled = disabled.get();
            for (WorkerHistoryItem change : changes) {
                if (change instanceof WorkerHistoryItem.TaskUpdate) {
                    TaskAnnouncement announcement = ((WorkerHistoryItem.TaskUpdate) change).getTaskAnnouncement();
                    newSnapshot.put(announcement.getTaskId(), announcement);
                    delta.add(announcement);
                } else if (change instanceof WorkerHistoryItem.Metadata) {
                    isWorkerDisabled = ((WorkerHistoryItem.Metadata) change).isDisabled();
                } else {
                    log.makeAlert("Got unknown sync update[%s] from worker[%s]. Ignored.", change.getClass().getName(), worker.getHost()).emit();
                }
            }
            for (TaskAnnouncement announcement : tasksSnapshotRef.get().values()) {
                if (!newSnapshot.containsKey(announcement.getTaskId()) && !announcement.getTaskStatus().isComplete()) {
                    log.warn("task[%s] in state[%s] suddenly disappeared on worker[%s]. failing it.", announcement.getTaskId(), announcement.getStatus(), worker.getHost());
                    delta.add(TaskAnnouncement.create(announcement.getTaskId(), announcement.getTaskType(), announcement.getTaskResource(), TaskStatus.failure(announcement.getTaskId(), "This task disappeared on the worker where it was assigned. " + "See overlord logs for more details."), announcement.getTaskLocation(), announcement.getTaskDataSource()));
                }
            }
            tasksSnapshotRef.set(newSnapshot);
            notifyListener(delta, isWorkerDisabled);
        }

        @Override
        public void deltaSync(List<WorkerHistoryItem> changes) {
            List<TaskAnnouncement> delta = new ArrayList<>();
            boolean isWorkerDisabled = disabled.get();
            for (WorkerHistoryItem change : changes) {
                if (change instanceof WorkerHistoryItem.TaskUpdate) {
                    TaskAnnouncement announcement = ((WorkerHistoryItem.TaskUpdate) change).getTaskAnnouncement();
                    tasksSnapshotRef.get().put(announcement.getTaskId(), announcement);
                    delta.add(announcement);
                } else if (change instanceof WorkerHistoryItem.TaskRemoval) {
                    String taskId = ((WorkerHistoryItem.TaskRemoval) change).getTaskId();
                    TaskAnnouncement announcement = tasksSnapshotRef.get().remove(taskId);
                    if (announcement != null && !announcement.getTaskStatus().isComplete()) {
                        log.warn("task[%s] in state[%s] suddenly disappeared on worker[%s]. failing it.", announcement.getTaskId(), announcement.getStatus(), worker.getHost());
                        delta.add(TaskAnnouncement.create(announcement.getTaskId(), announcement.getTaskType(), announcement.getTaskResource(), TaskStatus.failure(announcement.getTaskId(), "This task disappeared on the worker where it was assigned. " + "See overlord logs for more details."), announcement.getTaskLocation(), announcement.getTaskDataSource()));
                    }
                } else if (change instanceof WorkerHistoryItem.Metadata) {
                    isWorkerDisabled = ((WorkerHistoryItem.Metadata) change).isDisabled();
                } else {
                    log.makeAlert("Got unknown sync update[%s] from worker[%s]. Ignored.", change.getClass().getName(), worker.getHost()).emit();
                }
            }
            notifyListener(delta, isWorkerDisabled);
        }

        private void notifyListener(List<TaskAnnouncement> announcements, boolean isWorkerDisabled) {
            for (TaskAnnouncement announcement : announcements) {
                try {
                    listener.taskAddedOrUpdated(announcement, WorkerHolder.this);
                } catch (Exception ex) {
                    log.error(ex, "Unknown exception while updating task[%s] state from worker[%s].", announcement.getTaskId(), worker.getHost());
                }
            }
            if (isWorkerDisabled != disabled.get()) {
                disabled.set(isWorkerDisabled);
                log.info("Worker[%s] disabled set to [%s].", worker.getHost(), isWorkerDisabled);
            }
        }
    };
}

Example 2 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class RemoteTaskRunnerTestUtils method mockWorkerRunningTask.

void mockWorkerRunningTask(final String workerId, final Task task) throws Exception {
    cf.delete().forPath(JOINER.join(TASKS_PATH, workerId, task.getId()));
    final String taskStatusPath = JOINER.join(STATUS_PATH, workerId, task.getId());
    TaskAnnouncement taskAnnouncement = TaskAnnouncement.create(task, TaskStatus.running(task.getId()), DUMMY_LOCATION);
    cf.create().creatingParentsIfNeeded().forPath(taskStatusPath, jsonMapper.writeValueAsBytes(taskAnnouncement));
    Preconditions.checkNotNull(cf.checkExists().forPath(taskStatusPath), "Failed to write status on [%s]", taskStatusPath);
}

Example 3 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class RemoteTaskRunnerTestUtils method mockWorkerCompleteSuccessfulTask.

void mockWorkerCompleteSuccessfulTask(final String workerId, final Task task) throws Exception {
    TaskAnnouncement taskAnnouncement = TaskAnnouncement.create(task, TaskStatus.success(task.getId()), DUMMY_LOCATION);
    cf.setData().forPath(JOINER.join(STATUS_PATH, workerId, task.getId()), jsonMapper.writeValueAsBytes(taskAnnouncement));
}

Example 4 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class RemoteTaskRunnerTestUtils method mockWorkerCompleteFailedTask.

void mockWorkerCompleteFailedTask(final String workerId, final Task task) throws Exception {
    TaskAnnouncement taskAnnouncement = TaskAnnouncement.create(task, TaskStatus.failure(task.getId(), "Dummy task status failure for testing"), DUMMY_LOCATION);
    cf.setData().forPath(JOINER.join(STATUS_PATH, workerId, task.getId()), jsonMapper.writeValueAsBytes(taskAnnouncement));
}

Example 5 with TaskAnnouncement

use of org.apache.druid.indexing.worker.TaskAnnouncement in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method createWorkerHolder.

private static WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements, // running/completed on the worker.
List<TaskAnnouncement> preExistingTaskAnnouncements, // defines behavior for what to do when a particular task is assigned
Map<Task, List<TaskAnnouncement>> toBeAssignedTasks, // work completed
AtomicInteger ticks, // happened.
Set<String> actualShutdowns) {
    return new WorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements) {

        private final String workerHost;

        private final int workerPort;

        private final LifecycleLock startStopLock = new LifecycleLock();

        {
            String hostAndPort = worker.getHost();
            int colonIndex = hostAndPort.indexOf(':');
            if (colonIndex == -1) {
                throw new IAE("Invalid host and port: [%s]", colonIndex);
            }
            workerHost = hostAndPort.substring(0, colonIndex);
            workerPort = Integer.parseInt(hostAndPort.substring(colonIndex + 1));
        }

        @Override
        public void start() {
            synchronized (startStopLock) {
                if (!startStopLock.canStart()) {
                    throw new ISE("Can't start worker[%s:%s].", workerHost, workerPort);
                }
                try {
                    disabled.set(false);
                    if (!preExistingTaskAnnouncements.isEmpty()) {
                        workersSyncExec.execute(() -> {
                            for (TaskAnnouncement announcement : preExistingTaskAnnouncements) {
                                tasksSnapshotRef.get().put(announcement.getTaskId(), announcement);
                                listener.taskAddedOrUpdated(announcement, this);
                            }
                            ticks.incrementAndGet();
                        });
                    }
                    startStopLock.started();
                } finally {
                    startStopLock.exitStart();
                }
            }
        }

        @Override
        public void stop() {
            synchronized (startStopLock) {
                if (!startStopLock.canStop()) {
                    throw new ISE("Can't stop worker[%s:%s].", workerHost, workerPort);
                }
                try {
                } finally {
                    startStopLock.exitStop();
                }
            }
        }

        @Override
        public boolean isInitialized() {
            return true;
        }

        @Override
        public void waitForInitialization() {
        }

        @Override
        public boolean assignTask(Task task) {
            try {
                Thread.sleep(500);
            } catch (InterruptedException ex) {
                throw new RuntimeException(ex);
            }
            if (toImmutable().getCurrCapacityUsed() > worker.getCapacity()) {
                throw new ISE("Got assigned tasks more than capacity.");
            }
            final List<TaskAnnouncement> announcements;
            if (toBeAssignedTasks.containsKey(task)) {
                announcements = toBeAssignedTasks.get(task);
            } else {
                // no behavior specified for the task, so do default behavior of completing the task
                announcements = new ArrayList<>();
                announcements.add(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.unknown()));
                announcements.add(TaskAnnouncement.create(task, TaskStatus.running(task.getId()), TaskLocation.create(workerHost, workerPort, -1)));
                announcements.add(TaskAnnouncement.create(task, TaskStatus.success(task.getId()), TaskLocation.create(workerHost, workerPort, -1)));
            }
            workersSyncExec.execute(() -> {
                for (TaskAnnouncement announcement : announcements) {
                    try {
                        Thread.sleep(100);
                    } catch (InterruptedException ex) {
                        throw new RuntimeException(ex);
                    }
                    tasksSnapshotRef.get().put(announcement.getTaskId(), announcement);
                    listener.taskAddedOrUpdated(announcement, this);
                }
                ticks.incrementAndGet();
            });
            return true;
        }

        @Override
        public void shutdownTask(String taskId) {
            actualShutdowns.add(taskId);
        }
    };
}