Examples with ImmutableWorkerInfo io.druid.indexing.overlord.ImmutableWorkerInfo used on opensource projects

Example 21 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class PendingTaskBasedWorkerResourceManagementStrategy method getWorkersNeededToAssignTasks.

int getWorkersNeededToAssignTasks(final WorkerTaskRunnerConfig workerTaskRunnerConfig, final WorkerBehaviorConfig workerConfig, final Collection<Task> pendingTasks, final Collection<ImmutableWorkerInfo> workers) {
    final Collection<ImmutableWorkerInfo> validWorkers = Collections2.filter(workers, ResourceManagementUtil.createValidWorkerPredicate(config));
    Map<String, ImmutableWorkerInfo> workersMap = Maps.newHashMap();
    for (ImmutableWorkerInfo worker : validWorkers) {
        workersMap.put(worker.getWorker().getHost(), worker);
    }
    WorkerSelectStrategy workerSelectStrategy = workerConfig.getSelectStrategy();
    int need = 0;
    int capacity = getExpectedWorkerCapacity(workers);
    // the number of additional workers needed to assign all the pending tasks is noted
    for (Task task : pendingTasks) {
        Optional<ImmutableWorkerInfo> selectedWorker = workerSelectStrategy.findWorkerForTask(workerTaskRunnerConfig, ImmutableMap.copyOf(workersMap), task);
        final ImmutableWorkerInfo workerRunningTask;
        if (selectedWorker.isPresent()) {
            workerRunningTask = selectedWorker.get();
        } else {
            // None of the existing worker can run this task, we need to provision one worker for it.
            // create a dummy worker and try to simulate assigning task to it.
            workerRunningTask = createDummyWorker("dummy" + need, capacity, workerTaskRunnerConfig.getMinWorkerVersion());
            need++;
        }
        // Update map with worker running task
        workersMap.put(workerRunningTask.getWorker().getHost(), workerWithTask(workerRunningTask, task));
    }
    return need;
}

Example 22 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class SimpleWorkerResourceManagementStrategy method doTerminate.

boolean doTerminate(WorkerTaskRunner runner) {
    Collection<? extends TaskRunnerWorkItem> pendingTasks = runner.getPendingTasks();
    synchronized (lock) {
        final WorkerBehaviorConfig workerConfig = workerConfigRef.get();
        if (workerConfig == null) {
            log.warn("No workerConfig available, cannot terminate workers.");
            return false;
        }
        boolean didTerminate = false;
        final Set<String> workerNodeIds = Sets.newHashSet(workerConfig.getAutoScaler().ipToIdLookup(Lists.newArrayList(Iterables.transform(runner.getLazyWorkers(), new Function<Worker, String>() {

            @Override
            public String apply(Worker input) {
                return input.getIp();
            }
        }))));
        final Set<String> stillExisting = Sets.newHashSet();
        for (String s : currentlyTerminating) {
            if (workerNodeIds.contains(s)) {
                stillExisting.add(s);
            }
        }
        currentlyTerminating.clear();
        currentlyTerminating.addAll(stillExisting);
        Collection<ImmutableWorkerInfo> workers = getWorkers(runner);
        updateTargetWorkerCount(workerConfig, pendingTasks, workers);
        if (currentlyTerminating.isEmpty()) {
            final int excessWorkers = (workers.size() + currentlyProvisioning.size()) - targetWorkerCount;
            if (excessWorkers > 0) {
                final Predicate<ImmutableWorkerInfo> isLazyWorker = ResourceManagementUtil.createLazyWorkerPredicate(config);
                final Collection<String> laziestWorkerIps = Collections2.transform(runner.markWorkersLazy(isLazyWorker, excessWorkers), new Function<Worker, String>() {

                    @Override
                    public String apply(Worker worker) {
                        return worker.getIp();
                    }
                });
                if (laziestWorkerIps.isEmpty()) {
                    log.info("Wanted to terminate %,d workers, but couldn't find any lazy ones!", excessWorkers);
                } else {
                    log.info("Terminating %,d workers (wanted %,d): %s", laziestWorkerIps.size(), excessWorkers, Joiner.on(", ").join(laziestWorkerIps));
                    final AutoScalingData terminated = workerConfig.getAutoScaler().terminate(ImmutableList.copyOf(laziestWorkerIps));
                    if (terminated != null) {
                        currentlyTerminating.addAll(terminated.getNodeIds());
                        lastTerminateTime = new DateTime();
                        scalingStats.addTerminateEvent(terminated);
                        didTerminate = true;
                    }
                }
            }
        } else {
            Duration durSinceLastTerminate = new Duration(lastTerminateTime, new DateTime());
            log.info("%s terminating. Current wait time: %s", currentlyTerminating, durSinceLastTerminate);
            if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration().toStandardDuration())) {
                log.makeAlert("Worker node termination taking too long!").addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis()).addData("terminatingCount", currentlyTerminating.size()).emit();
                currentlyTerminating.clear();
            }
        }
        return didTerminate;
    }
}

Example 23 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class FillCapacityWorkerSelectStrategy method findWorkerForTask.

@Override
public Optional<ImmutableWorkerInfo> findWorkerForTask(final WorkerTaskRunnerConfig config, final ImmutableMap<String, ImmutableWorkerInfo> zkWorkers, final Task task) {
    TreeSet<ImmutableWorkerInfo> sortedWorkers = Sets.newTreeSet(new Comparator<ImmutableWorkerInfo>() {

        @Override
        public int compare(ImmutableWorkerInfo zkWorker, ImmutableWorkerInfo zkWorker2) {
            int retVal = Ints.compare(zkWorker2.getCurrCapacityUsed(), zkWorker.getCurrCapacityUsed());
            if (retVal == 0) {
                retVal = zkWorker.getWorker().getVersion().compareTo(zkWorker2.getWorker().getVersion());
            }
            return retVal;
        }
    });
    sortedWorkers.addAll(zkWorkers.values());
    final String minWorkerVer = config.getMinWorkerVersion();
    for (ImmutableWorkerInfo zkWorker : sortedWorkers) {
        if (zkWorker.canRunTask(task) && zkWorker.isValidVersion(minWorkerVer)) {
            return Optional.of(zkWorker);
        }
    }
    return Optional.absent();
}

Example 24 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class JavaScriptWorkerSelectStrategyTest method testFindWorkerForTask.

@Test
public void testFindWorkerForTask() {
    ImmutableWorkerInfo worker1 = createMockWorker(1, true, true);
    ImmutableWorkerInfo worker2 = createMockWorker(1, true, true);
    ImmutableMap<String, ImmutableWorkerInfo> workerMap = ImmutableMap.of("10.0.0.1", worker1, "10.0.0.3", worker2);
    ImmutableWorkerInfo workerForBatchTask = STRATEGY.findWorkerForTask(new TestRemoteTaskRunnerConfig(new Period("PT1S")), workerMap, createMockTask("index_hadoop")).get();
    // batch tasks should be sent to worker1
    Assert.assertEquals(worker1, workerForBatchTask);
    ImmutableWorkerInfo workerForOtherTask = STRATEGY.findWorkerForTask(new TestRemoteTaskRunnerConfig(new Period("PT1S")), workerMap, createMockTask("other_type")).get();
    // all other tasks should be sent to worker2
    Assert.assertEquals(worker2, workerForOtherTask);
}

Example 25 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class SimpleResourceManagementStrategyTest method testNoActionNeeded.

@Test
public void testNoActionNeeded() throws Exception {
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.newArrayList("ip"));
    EasyMock.replay(autoScaler);
    RemoteTaskRunner runner = EasyMock.createMock(RemoteTaskRunner.class);
    EasyMock.expect(runner.getPendingTasks()).andReturn(Collections.singletonList(new RemoteTaskRunnerWorkItem(testTask.getId(), null, null).withQueueInsertionTime(new DateTime()))).times(2);
    EasyMock.expect(runner.getWorkers()).andReturn(Arrays.asList(new TestZkWorker(NoopTask.create()).toImmutable(), new TestZkWorker(NoopTask.create()).toImmutable())).times(2);
    EasyMock.expect(runner.getLazyWorkers()).andReturn(Lists.<Worker>newArrayList());
    EasyMock.expect(runner.markWorkersLazy(EasyMock.<Predicate<ImmutableWorkerInfo>>anyObject(), EasyMock.anyInt())).andReturn(Collections.<Worker>emptyList());
    EasyMock.replay(runner);
    boolean terminatedSomething = simpleResourceManagementStrategy.doTerminate(runner);
    Assert.assertFalse(terminatedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.reset(autoScaler);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(2);
    EasyMock.expect(autoScaler.ipToIdLookup(EasyMock.<List<String>>anyObject())).andReturn(Lists.newArrayList("ip"));
    EasyMock.replay(autoScaler);
    boolean provisionedSomething = simpleResourceManagementStrategy.doProvision(runner);
    Assert.assertFalse(provisionedSomething);
    EasyMock.verify(autoScaler);
    EasyMock.verify(runner);
}