Examples with ImmutableWorkerInfo io.druid.indexing.overlord.ImmutableWorkerInfo used on opensource projects

Example 16 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class FillCapacityWithAffinityWorkerSelectStrategyTest method testFindWorkerForTask.

@Test
public void testFindWorkerForTask() throws Exception {
    FillCapacityWorkerSelectStrategy strategy = new FillCapacityWithAffinityWorkerSelectStrategy(new FillCapacityWithAffinityConfig(ImmutableMap.of("foo", Arrays.asList("localhost"))));
    Optional<ImmutableWorkerInfo> optional = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("lhost", "lhost", 1, "v1"), 0, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now()), "localhost", new ImmutableWorkerInfo(new Worker("localhost", "localhost", 1, "v1"), 0, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now())), new NoopTask(null, 1, 0, null, null, null) {

        @Override
        public String getDataSource() {
            return "foo";
        }
    });
    ImmutableWorkerInfo worker = optional.get();
    Assert.assertEquals("localhost", worker.getWorker().getHost());
}

Example 17 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class FillCapacityWithAffinityWorkerSelectStrategyTest method testFindWorkerForTaskWithNulls.

@Test
public void testFindWorkerForTaskWithNulls() throws Exception {
    FillCapacityWorkerSelectStrategy strategy = new FillCapacityWithAffinityWorkerSelectStrategy(new FillCapacityWithAffinityConfig(ImmutableMap.of("foo", Arrays.asList("localhost"))));
    Optional<ImmutableWorkerInfo> optional = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("lhost", new ImmutableWorkerInfo(new Worker("lhost", "lhost", 1, "v1"), 0, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now()), "localhost", new ImmutableWorkerInfo(new Worker("localhost", "localhost", 1, "v1"), 0, Sets.<String>newHashSet(), Sets.<String>newHashSet(), DateTime.now())), new NoopTask(null, 1, 0, null, null, null));
    ImmutableWorkerInfo worker = optional.get();
    Assert.assertEquals("lhost", worker.getWorker().getHost());
}

Example 18 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class JavaScriptWorkerSelectStrategyTest method testNoValidWorker.

@Test
public void testNoValidWorker() {
    ImmutableMap<String, ImmutableWorkerInfo> workerMap = ImmutableMap.of("10.0.0.1", createMockWorker(1, true, false), "10.0.0.4", createMockWorker(1, true, false));
    Optional<ImmutableWorkerInfo> workerForBatchTask = STRATEGY.findWorkerForTask(new TestRemoteTaskRunnerConfig(new Period("PT1S")), workerMap, createMockTask("index_hadoop"));
    Assert.assertFalse(workerForBatchTask.isPresent());
    Optional<ImmutableWorkerInfo> workerForOtherTask = STRATEGY.findWorkerForTask(new TestRemoteTaskRunnerConfig(new Period("PT1S")), workerMap, createMockTask("otherTask"));
    // all other tasks should be sent to worker2
    Assert.assertFalse(workerForOtherTask.isPresent());
}

Example 19 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class JavaScriptWorkerSelectStrategyTest method testIsolationOfBatchWorker.

@Test
public void testIsolationOfBatchWorker() {
    ImmutableMap<String, ImmutableWorkerInfo> workerMap = ImmutableMap.of("10.0.0.1", createMockWorker(1, true, true), "10.0.0.2", createMockWorker(1, true, true));
    Optional<ImmutableWorkerInfo> workerForOtherTask = STRATEGY.findWorkerForTask(new TestRemoteTaskRunnerConfig(new Period("PT1S")), workerMap, createMockTask("other_type"));
    Assert.assertFalse(workerForOtherTask.isPresent());
}

Example 20 with ImmutableWorkerInfo

use of io.druid.indexing.overlord.ImmutableWorkerInfo in project druid by druid-io.

the class PendingTaskBasedWorkerResourceManagementStrategy method doProvision.

@Override
public boolean doProvision(WorkerTaskRunner runner) {
    Collection<Task> pendingTasks = runner.getPendingTaskPayloads();
    Collection<ImmutableWorkerInfo> workers = runner.getWorkers();
    synchronized (lock) {
        boolean didProvision = false;
        final WorkerBehaviorConfig workerConfig = workerConfigRef.get();
        if (workerConfig == null || workerConfig.getAutoScaler() == null) {
            log.error("No workerConfig available, cannot provision new workers.");
            return false;
        }
        final Collection<String> workerNodeIds = getWorkerNodeIDs(Collections2.transform(workers, new Function<ImmutableWorkerInfo, Worker>() {

            @Override
            public Worker apply(ImmutableWorkerInfo input) {
                return input.getWorker();
            }
        }), workerConfig);
        currentlyProvisioning.removeAll(workerNodeIds);
        if (currentlyProvisioning.isEmpty()) {
            int want = getScaleUpNodeCount(runner.getConfig(), workerConfig, pendingTasks, workers);
            while (want > 0) {
                final AutoScalingData provisioned = workerConfig.getAutoScaler().provision();
                final List<String> newNodes = provisioned == null ? ImmutableList.<String>of() : provisioned.getNodeIds();
                if (newNodes.isEmpty()) {
                    log.warn("NewNodes is empty, returning from provision loop");
                    break;
                } else {
                    currentlyProvisioning.addAll(newNodes);
                    lastProvisionTime = new DateTime();
                    scalingStats.addProvisionEvent(provisioned);
                    want -= provisioned.getNodeIds().size();
                    didProvision = true;
                }
            }
        } else {
            Duration durSinceLastProvision = new Duration(lastProvisionTime, new DateTime());
            log.info("%s provisioning. Current wait time: %s", currentlyProvisioning, durSinceLastProvision);
            if (durSinceLastProvision.isLongerThan(config.getMaxScalingDuration().toStandardDuration())) {
                log.makeAlert("Worker node provisioning taking too long!").addData("millisSinceLastProvision", durSinceLastProvision.getMillis()).addData("provisioningCount", currentlyProvisioning.size()).emit();
                workerConfig.getAutoScaler().terminateWithIds(Lists.newArrayList(currentlyProvisioning));
                currentlyProvisioning.clear();
            }
        }
        return didProvision;
    }
}