Examples with ImmutableWorkerInfo org.apache.druid.indexing.overlord.ImmutableWorkerInfo used on opensource projects

Example 26 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class EqualDistributionWithCategorySpecWorkerSelectStrategyTest method selectWorker.

private ImmutableWorkerInfo selectWorker(WorkerCategorySpec workerCategorySpec) {
    final EqualDistributionWithCategorySpecWorkerSelectStrategy strategy = new EqualDistributionWithCategorySpecWorkerSelectStrategy(workerCategorySpec);
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), WORKERS_FOR_TIER_TESTS, new NoopTask(null, null, "ds1", 1, 0, null, null, null));
    return worker;
}

Example 27 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class FillCapacityWithAffinityWorkerSelectStrategyTest method testIsolation.

@Test
public void testIsolation() {
    FillCapacityWorkerSelectStrategy strategy = new FillCapacityWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost")), false));
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost", new ImmutableWorkerInfo(new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null));
    Assert.assertNull(worker);
}

Example 28 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class JavaScriptWorkerSelectStrategyTest method createMockWorker.

private ImmutableWorkerInfo createMockWorker(int currCapacityUsed, boolean canRunTask, boolean isValidVersion) {
    ImmutableWorkerInfo worker = EasyMock.createMock(ImmutableWorkerInfo.class);
    EasyMock.expect(worker.canRunTask(EasyMock.anyObject(Task.class), EasyMock.anyDouble())).andReturn(canRunTask).anyTimes();
    EasyMock.expect(worker.getCurrCapacityUsed()).andReturn(currCapacityUsed).anyTimes();
    EasyMock.expect(worker.getCurrParallelIndexCapacityUsed()).andReturn(0).anyTimes();
    EasyMock.expect(worker.isValidVersion(EasyMock.anyString())).andReturn(isValidVersion).anyTimes();
    EasyMock.replay(worker);
    return worker;
}

Example 29 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class HttpRemoteTaskRunner method pendingTasksExecutionLoop.

private void pendingTasksExecutionLoop() {
    while (!Thread.interrupted() && lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)) {
        try {
            // Find one pending task to run and a worker to run on
            HttpRemoteTaskRunnerWorkItem taskItem = null;
            ImmutableWorkerInfo immutableWorker = null;
            synchronized (statusLock) {
                Iterator<String> iter = pendingTaskIds.iterator();
                while (iter.hasNext()) {
                    String taskId = iter.next();
                    HttpRemoteTaskRunnerWorkItem ti = tasks.get(taskId);
                    if (ti == null || !ti.getState().isPending()) {
                        // happens if the task was shutdown, failed or observed running by a worker
                        iter.remove();
                        continue;
                    }
                    if (ti.getState() == HttpRemoteTaskRunnerWorkItem.State.PENDING_WORKER_ASSIGN) {
                        // run it on a worker, skip to next.
                        continue;
                    }
                    if (ti.getTask() == null) {
                        // this is not supposed to happen except for a bug, we want to mark this task failed but
                        // taskComplete(..) can not be called while holding statusLock. See the javadoc on that
                        // method.
                        // so this will get marked failed afterwards outside of current synchronized block.
                        taskItem = ti;
                        break;
                    }
                    immutableWorker = findWorkerToRunTask(ti.getTask());
                    if (immutableWorker == null) {
                        continue;
                    }
                    String prevUnackedTaskId = workersWithUnacknowledgedTask.putIfAbsent(immutableWorker.getWorker().getHost(), taskId);
                    if (prevUnackedTaskId != null) {
                        log.makeAlert("Found worker[%s] with unacked task[%s] but still was identified to run task[%s].", immutableWorker.getWorker().getHost(), prevUnackedTaskId, taskId).emit();
                    }
                    // set state to PENDING_WORKER_ASSIGN before releasing the lock so that this task item is not picked
                    // up by another task execution thread.
                    // note that we can't simply delete this task item from pendingTaskIds or else we would have to add it
                    // back if this thread couldn't run this task for any reason, which we will know at some later time
                    // and also we will need to add it back to its old position in the list. that becomes complex quickly.
                    // Instead we keep the PENDING_WORKER_ASSIGN to notify other task execution threads not to pick this one up.
                    // And, it is automatically removed by any of the task execution threads when they notice that
                    // ti.getState().isPending() is false (at the beginning of this loop)
                    ti.setState(HttpRemoteTaskRunnerWorkItem.State.PENDING_WORKER_ASSIGN);
                    taskItem = ti;
                    break;
                }
                if (taskItem == null) {
                    // Either no pending task is found or no suitable worker is found for any of the pending tasks.
                    // statusLock.notifyAll() is called whenever a new task shows up or if there is a possibility for a task
                    // to successfully get worker to run, for example when a new worker shows up, a task slot opens up
                    // because some task completed etc.
                    statusLock.wait(TimeUnit.MINUTES.toMillis(1));
                    continue;
                }
            }
            String taskId = taskItem.getTaskId();
            if (taskItem.getTask() == null) {
                log.makeAlert("No Task obj found in TaskItem for taskID[%s]. Failed.", taskId).emit();
                // taskComplete(..) must be called outside of statusLock, see comments on method.
                taskComplete(taskItem, null, TaskStatus.failure(taskId, "No payload found for this task. " + "See overlord logs and middleManager/indexer logs for more details."));
                continue;
            }
            if (immutableWorker == null) {
                throw new ISE("Unexpected state: null immutableWorker");
            }
            try {
                // this will send HTTP request to worker for assigning task
                if (!runTaskOnWorker(taskItem, immutableWorker.getWorker().getHost())) {
                    if (taskItem.getState() == HttpRemoteTaskRunnerWorkItem.State.PENDING_WORKER_ASSIGN) {
                        taskItem.revertStateFromPendingWorkerAssignToPending();
                    }
                }
            } catch (InterruptedException ex) {
                log.info("Got InterruptedException while assigning task[%s].", taskId);
                throw ex;
            } catch (Throwable th) {
                log.makeAlert(th, "Exception while trying to assign task").addData("taskId", taskId).emit();
                // taskComplete(..) must be called outside of statusLock, see comments on method.
                taskComplete(taskItem, null, TaskStatus.failure(taskId, "Failed to assign this task. See overlord logs for more details."));
            } finally {
                synchronized (statusLock) {
                    workersWithUnacknowledgedTask.remove(immutableWorker.getWorker().getHost());
                    statusLock.notifyAll();
                }
            }
        } catch (InterruptedException ex) {
            log.info("Interrupted, will Exit.");
            Thread.currentThread().interrupt();
        } catch (Throwable th) {
            log.makeAlert(th, "Unknown Exception while trying to assign tasks.").emit();
        }
    }
}

Example 30 with ImmutableWorkerInfo

use of org.apache.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class HttpRemoteTaskRunner method getTotalTaskSlotCount.

@Override
public Map<String, Long> getTotalTaskSlotCount() {
    Map<String, Long> totalPeons = new HashMap<>();
    for (ImmutableWorkerInfo worker : getWorkers()) {
        String workerCategory = worker.getWorker().getCategory();
        int workerCapacity = worker.getWorker().getCapacity();
        totalPeons.compute(workerCategory, (category, totalCapacity) -> totalCapacity == null ? workerCapacity : totalCapacity + workerCapacity);
    }
    return totalPeons;
}