Search in sources :

Example 31 with DruidNode

use of org.apache.druid.server.DruidNode in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testWorkerDisapperAndReappearBeforeItsCleanup.

@Test(timeout = 60_000L)
public void testWorkerDisapperAndReappearBeforeItsCleanup() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            if (workerHolders.containsKey(worker.getHost())) {
                return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
            } else {
                throw new ISE("No WorkerHolder for [%s].", worker.getHost());
            }
        }
    };
    taskRunner.start();
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    DiscoveryDruidNode druidNode = new DiscoveryDruidNode(new DruidNode("service", "host", false, 1234, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task1, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("host", 1234, 1235))), task2, ImmutableList.of(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)))), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    Future<TaskStatus> future1 = taskRunner.run(task1);
    Future<TaskStatus> future2 = taskRunner.run(task2);
    while (taskRunner.getPendingTasks().size() > 0) {
        Thread.sleep(100);
    }
    druidNodeDiscovery.getListeners().get(0).nodesRemoved(ImmutableList.of(druidNode));
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task2, TaskStatus.success(task2.getId()), TaskLocation.create("host", 1234, 1235))), ImmutableMap.of(), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    Assert.assertTrue(future1.get().isSuccess());
    Assert.assertTrue(future2.get().isSuccess());
}
Also used : IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) Task(org.apache.druid.indexing.common.task.Task) NoopTask(org.apache.druid.indexing.common.task.NoopTask) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) CuratorFramework(org.apache.curator.framework.CuratorFramework) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) Worker(org.apache.druid.indexing.worker.Worker) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ISE(org.apache.druid.java.util.common.ISE) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AtomicReference(java.util.concurrent.atomic.AtomicReference) TaskStatus(org.apache.druid.indexer.TaskStatus) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) HttpClient(org.apache.druid.java.util.http.client.HttpClient) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) Test(org.junit.Test)

Example 32 with DruidNode

use of org.apache.druid.server.DruidNode in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testWorkerDisapperAndReappearAfterItsCleanup.

@Test(timeout = 60_000L)
public void testWorkerDisapperAndReappearAfterItsCleanup() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public Period getTaskCleanupTimeout() {
            return Period.millis(1);
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            if (workerHolders.containsKey(worker.getHost())) {
                return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
            } else {
                throw new ISE("No WorkerHolder for [%s].", worker.getHost());
            }
        }
    };
    taskRunner.start();
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    DiscoveryDruidNode druidNode = new DiscoveryDruidNode(new DruidNode("service", "host", false, 1234, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task1, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("host", 1234, 1235))), task2, ImmutableList.of(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)))), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    Future<TaskStatus> future1 = taskRunner.run(task1);
    Future<TaskStatus> future2 = taskRunner.run(task2);
    while (taskRunner.getPendingTasks().size() > 0) {
        Thread.sleep(100);
    }
    druidNodeDiscovery.getListeners().get(0).nodesRemoved(ImmutableList.of(druidNode));
    Assert.assertTrue(future1.get().isFailure());
    Assert.assertTrue(future2.get().isFailure());
    Assert.assertNotNull(future1.get().getErrorMsg());
    Assert.assertNotNull(future2.get().getErrorMsg());
    Assert.assertTrue(future1.get().getErrorMsg().startsWith("The worker that this task was assigned disappeared and did not report cleanup within timeout"));
    Assert.assertTrue(future2.get().getErrorMsg().startsWith("The worker that this task was assigned disappeared and did not report cleanup within timeout"));
    AtomicInteger ticks = new AtomicInteger();
    Set<String> actualShutdowns = new ConcurrentHashSet<>();
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235))), ImmutableMap.of(), ticks, actualShutdowns));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    while (ticks.get() < 1) {
        Thread.sleep(100);
    }
    Assert.assertEquals(ImmutableSet.of(task2.getId()), actualShutdowns);
    Assert.assertTrue(taskRunner.run(task1).get().isFailure());
    Assert.assertTrue(taskRunner.run(task2).get().isFailure());
}
Also used : IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) Task(org.apache.druid.indexing.common.task.Task) NoopTask(org.apache.druid.indexing.common.task.NoopTask) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) CuratorFramework(org.apache.curator.framework.CuratorFramework) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) ConcurrentHashSet(org.eclipse.jetty.util.ConcurrentHashSet) Worker(org.apache.druid.indexing.worker.Worker) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ISE(org.apache.druid.java.util.common.ISE) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Period(org.joda.time.Period) AtomicReference(java.util.concurrent.atomic.AtomicReference) TaskStatus(org.apache.druid.indexer.TaskStatus) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) HttpClient(org.apache.druid.java.util.http.client.HttpClient) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) Test(org.junit.Test)

Example 33 with DruidNode

use of org.apache.druid.server.DruidNode in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testFreshStartAndStop.

/*
  Simulates startup of Overlord. Overlord is then stopped and is expected to close down certain things.
   */
@Test(timeout = 60_000L)
public void testFreshStartAndStop() {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery).times(2);
    ProvisioningStrategy provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    ProvisioningService provisioningService = EasyMock.createNiceMock(ProvisioningService.class);
    EasyMock.expect(provisioningStrategy.makeProvisioningService(isA(HttpRemoteTaskRunner.class))).andReturn(provisioningService);
    provisioningService.close();
    EasyMock.expectLastCall();
    EasyMock.replay(druidNodeDiscoveryProvider, provisioningStrategy, provisioningService);
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    HttpRemoteTaskRunner taskRunner = newHttpTaskRunnerInstance(druidNodeDiscoveryProvider, provisioningStrategy);
    taskRunner.start();
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1, druidNode2));
    ConcurrentMap<String, WorkerHolder> workers = taskRunner.getWorkersForTestingReadOnly();
    Assert.assertEquals(2, workers.size());
    Assert.assertTrue(workers.values().stream().noneMatch(w -> w.getUnderlyingSyncer().isExecutorShutdown()));
    workers.values().iterator().next().stop();
    taskRunner.stop();
    Assert.assertTrue(druidNodeDiscovery.getListeners().isEmpty());
    Assert.assertEquals(2, workers.size());
    Assert.assertTrue(workers.values().stream().allMatch(w -> w.getUnderlyingSyncer().isExecutorShutdown()));
    EasyMock.verify(druidNodeDiscoveryProvider, provisioningStrategy, provisioningService);
}
Also used : DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) EasyMock.isA(org.easymock.EasyMock.isA) ConcurrentHashSet(org.eclipse.jetty.util.ConcurrentHashSet) Future(java.util.concurrent.Future) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Optional(com.google.common.base.Optional) Task(org.apache.druid.indexing.common.task.Task) Map(java.util.Map) NoopProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.NoopProvisioningStrategy) TaskAnnouncement(org.apache.druid.indexing.worker.TaskAnnouncement) IAE(org.apache.druid.java.util.common.IAE) ImmutableSet(com.google.common.collect.ImmutableSet) Execs(org.apache.druid.java.util.common.concurrent.Execs) ImmutableMap(com.google.common.collect.ImmutableMap) DSuppliers(org.apache.druid.common.guava.DSuppliers) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StringUtils(org.apache.druid.java.util.common.StringUtils) Set(java.util.Set) ISE(org.apache.druid.java.util.common.ISE) TaskState(org.apache.druid.indexer.TaskState) NoopTask(org.apache.druid.indexing.common.task.NoopTask) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) NodeRole(org.apache.druid.discovery.NodeRole) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) Iterables(com.google.common.collect.Iterables) HttpClient(org.apache.druid.java.util.http.client.HttpClient) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) AtomicReference(java.util.concurrent.atomic.AtomicReference) TaskStatus(org.apache.druid.indexer.TaskStatus) Worker(org.apache.druid.indexing.worker.Worker) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) DruidNodeDiscovery(org.apache.druid.discovery.DruidNodeDiscovery) ImmutableList(com.google.common.collect.ImmutableList) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Predicates(com.google.common.base.Predicates) DefaultWorkerBehaviorConfig(org.apache.druid.indexing.overlord.setup.DefaultWorkerBehaviorConfig) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) ProvisioningService(org.apache.druid.indexing.overlord.autoscaling.ProvisioningService) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) Before(org.junit.Before) Period(org.joda.time.Period) WorkerConfig(org.apache.druid.indexing.worker.config.WorkerConfig) EmittingLogger(org.apache.druid.java.util.emitter.EmittingLogger) IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) TaskLocation(org.apache.druid.indexer.TaskLocation) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) ProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy) TestHelper(org.apache.druid.segment.TestHelper) LifecycleLock(org.apache.druid.concurrent.LifecycleLock) DruidNode(org.apache.druid.server.DruidNode) TaskRunnerWorkItem(org.apache.druid.indexing.overlord.TaskRunnerWorkItem) Assert(org.junit.Assert) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) ProvisioningService(org.apache.druid.indexing.overlord.autoscaling.ProvisioningService) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) NoopProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.NoopProvisioningStrategy) ProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy) Test(org.junit.Test)

Example 34 with DruidNode

use of org.apache.druid.server.DruidNode in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testMarkWorkersLazy.

@Test(timeout = 60_000L)
public void testMarkWorkersLazy() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    String additionalWorkerCategory = "category2";
    ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            if (workerHolders.containsKey(worker.getHost())) {
                return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
            } else {
                throw new ISE("No WorkerHolder for [%s].", worker.getHost());
            }
        }
    };
    taskRunner.start();
    Assert.assertTrue(taskRunner.getTotalTaskSlotCount().isEmpty());
    Assert.assertTrue(taskRunner.getIdleTaskSlotCount().isEmpty());
    Assert.assertTrue(taskRunner.getUsedTaskSlotCount().isEmpty());
    AtomicInteger ticks = new AtomicInteger();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host1:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task1, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("host1", 8080, -1)))), ticks, ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    taskRunner.run(task1);
    while (ticks.get() < 1) {
        Thread.sleep(100);
    }
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", additionalWorkerCategory)));
    workerHolders.put("host2:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task2, ImmutableList.of()), ticks, ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode2));
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    taskRunner.run(task2);
    while (ticks.get() < 2) {
        Thread.sleep(100);
    }
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    DiscoveryDruidNode druidNode3 = new DiscoveryDruidNode(new DruidNode("service", "host3", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host3:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode3));
    Assert.assertEquals(2, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(task1.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    Assert.assertEquals(task2.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
    Assert.assertEquals("host3:8080", Iterables.getOnlyElement(taskRunner.markWorkersLazy(Predicates.alwaysTrue(), Integer.MAX_VALUE)).getHost());
    Assert.assertEquals(2, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getLazyTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(additionalWorkerCategory));
}
Also used : IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) Task(org.apache.druid.indexing.common.task.Task) NoopTask(org.apache.druid.indexing.common.task.NoopTask) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) CuratorFramework(org.apache.curator.framework.CuratorFramework) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) Worker(org.apache.druid.indexing.worker.Worker) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ISE(org.apache.druid.java.util.common.ISE) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AtomicReference(java.util.concurrent.atomic.AtomicReference) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) HttpClient(org.apache.druid.java.util.http.client.HttpClient) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) Test(org.junit.Test)

Example 35 with DruidNode

use of org.apache.druid.server.DruidNode in project druid by druid-io.

the class OverlordTest method setUp.

@Before
public void setUp() throws Exception {
    req = EasyMock.createMock(HttpServletRequest.class);
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes();
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(new AuthenticationResult("druid", "druid", null, null)).anyTimes();
    req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().anyTimes();
    supervisorManager = EasyMock.createMock(SupervisorManager.class);
    taskLockbox = EasyMock.createStrictMock(TaskLockbox.class);
    taskLockbox.syncFromStorage();
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.add(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    // for second Noop Task directly added to deep storage.
    taskLockbox.add(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskActionClientFactory = EasyMock.createStrictMock(TaskActionClientFactory.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.anyObject())).andReturn(null).anyTimes();
    EasyMock.replay(taskLockbox, taskActionClientFactory, req);
    taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
    runTaskCountDownLatches = new CountDownLatch[2];
    runTaskCountDownLatches[0] = new CountDownLatch(1);
    runTaskCountDownLatches[1] = new CountDownLatch(1);
    taskCompletionCountDownLatches = new CountDownLatch[2];
    taskCompletionCountDownLatches[0] = new CountDownLatch(1);
    taskCompletionCountDownLatches[1] = new CountDownLatch(1);
    announcementLatch = new CountDownLatch(1);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    ServiceEmitter serviceEmitter = new NoopServiceEmitter();
    taskMaster = new TaskMaster(new TaskLockConfig(), new TaskQueueConfig(null, new Period(1), null, new Period(10)), new DefaultTaskConfig(), taskLockbox, taskStorage, taskActionClientFactory, druidNode, new TaskRunnerFactory<MockTaskRunner>() {

        @Override
        public MockTaskRunner build() {
            return new MockTaskRunner(runTaskCountDownLatches, taskCompletionCountDownLatches);
        }
    }, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            announcementLatch.countDown();
        }
    }, new CoordinatorOverlordServiceConfig(null, null), serviceEmitter, supervisorManager, EasyMock.createNiceMock(OverlordHelperManager.class), new TestDruidLeaderSelector());
    EmittingLogger.registerEmitter(serviceEmitter);
}
Also used : ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) CoordinatorOverlordServiceConfig(org.apache.druid.server.coordinator.CoordinatorOverlordServiceConfig) TaskStorageConfig(org.apache.druid.indexing.common.config.TaskStorageConfig) HeapMemoryTaskStorage(org.apache.druid.indexing.overlord.HeapMemoryTaskStorage) TaskActionClientFactory(org.apache.druid.indexing.common.actions.TaskActionClientFactory) Period(org.joda.time.Period) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) DefaultTaskConfig(org.apache.druid.indexing.overlord.config.DefaultTaskConfig) CountDownLatch(java.util.concurrent.CountDownLatch) TaskLockConfig(org.apache.druid.indexing.overlord.config.TaskLockConfig) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) HttpServletRequest(javax.servlet.http.HttpServletRequest) SupervisorManager(org.apache.druid.indexing.overlord.supervisor.SupervisorManager) TaskLockbox(org.apache.druid.indexing.overlord.TaskLockbox) TaskQueueConfig(org.apache.druid.indexing.overlord.config.TaskQueueConfig) DruidNode(org.apache.druid.server.DruidNode) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) TaskRunnerFactory(org.apache.druid.indexing.overlord.TaskRunnerFactory) Before(org.junit.Before)

Aggregations

DruidNode (org.apache.druid.server.DruidNode)61 Test (org.junit.Test)41 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)25 Self (org.apache.druid.guice.annotations.Self)19 ImmutableList (com.google.common.collect.ImmutableList)18 List (java.util.List)18 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)16 ZkPathsConfig (org.apache.druid.server.initialization.ZkPathsConfig)16 Binder (com.google.inject.Binder)15 Injector (com.google.inject.Injector)15 AtomicReference (java.util.concurrent.atomic.AtomicReference)15 DiscoveryDruidNode (org.apache.druid.discovery.DiscoveryDruidNode)14 HashSet (java.util.HashSet)12 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)12 DruidNodeDiscoveryProvider (org.apache.druid.discovery.DruidNodeDiscoveryProvider)12 TaskStatus (org.apache.druid.indexer.TaskStatus)12 Module (com.google.inject.Module)11 CuratorFramework (org.apache.curator.framework.CuratorFramework)11 NoopTask (org.apache.druid.indexing.common.task.NoopTask)11 Task (org.apache.druid.indexing.common.task.Task)11