Examples with Worker org.apache.druid.indexing.worker.Worker used on opensource projects

Example 36 with Worker

use of org.apache.druid.indexing.worker.Worker in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testMarkWorkersLazy.

@Test(timeout = 60_000L)
public void testMarkWorkersLazy() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    String additionalWorkerCategory = "category2";
    ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            if (workerHolders.containsKey(worker.getHost())) {
                return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
            } else {
                throw new ISE("No WorkerHolder for [%s].", worker.getHost());
            }
        }
    };
    taskRunner.start();
    Assert.assertTrue(taskRunner.getTotalTaskSlotCount().isEmpty());
    Assert.assertTrue(taskRunner.getIdleTaskSlotCount().isEmpty());
    Assert.assertTrue(taskRunner.getUsedTaskSlotCount().isEmpty());
    AtomicInteger ticks = new AtomicInteger();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host1:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task1, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("host1", 8080, -1)))), ticks, ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1));
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    taskRunner.run(task1);
    while (ticks.get() < 1) {
        Thread.sleep(100);
    }
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", additionalWorkerCategory)));
    workerHolders.put("host2:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task2, ImmutableList.of()), ticks, ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode2));
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    taskRunner.run(task2);
    while (ticks.get() < 2) {
        Thread.sleep(100);
    }
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    DiscoveryDruidNode druidNode3 = new DiscoveryDruidNode(new DruidNode("service", "host3", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host3:8080", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode3));
    Assert.assertEquals(2, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(WorkerConfig.DEFAULT_CATEGORY));
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(task1.getId(), Iterables.getOnlyElement(taskRunner.getRunningTasks()).getTaskId());
    Assert.assertEquals(task2.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
    Assert.assertEquals("host3:8080", Iterables.getOnlyElement(taskRunner.markWorkersLazy(Predicates.alwaysTrue(), Integer.MAX_VALUE)).getHost());
    Assert.assertEquals(2, taskRunner.getTotalTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(1, taskRunner.getTotalTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(0, taskRunner.getIdleTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getIdleTaskSlotCount().containsKey(additionalWorkerCategory));
    Assert.assertEquals(1, taskRunner.getUsedTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertEquals(0, taskRunner.getUsedTaskSlotCount().get(additionalWorkerCategory).longValue());
    Assert.assertEquals(1, taskRunner.getLazyTaskSlotCount().get(WorkerConfig.DEFAULT_CATEGORY).longValue());
    Assert.assertFalse(taskRunner.getLazyTaskSlotCount().containsKey(additionalWorkerCategory));
}

Example 37 with Worker

use of org.apache.druid.indexing.worker.Worker in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testTaskAddedOrUpdated3.

/*
   * Notifications received for tasks not known to TaskRunner maybe known to TaskStorage.
   * This could happen when TaskRunner starts and workers reports running/completed tasks on them.
   */
@Test
public void testTaskAddedOrUpdated3() {
    Task task1 = NoopTask.create("task1");
    Task task2 = NoopTask.create("task2");
    Task task3 = NoopTask.create("task3");
    Task task4 = NoopTask.create("task4");
    Task task5 = NoopTask.create("task5");
    Task task6 = NoopTask.create("task6");
    TaskStorage taskStorage = EasyMock.createMock(TaskStorage.class);
    EasyMock.expect(taskStorage.getStatus(task1.getId())).andReturn(Optional.of(TaskStatus.running(task1.getId())));
    EasyMock.expect(taskStorage.getStatus(task2.getId())).andReturn(Optional.of(TaskStatus.running(task2.getId())));
    EasyMock.expect(taskStorage.getStatus(task3.getId())).andReturn(Optional.of(TaskStatus.success(task3.getId())));
    EasyMock.expect(taskStorage.getStatus(task4.getId())).andReturn(Optional.of(TaskStatus.success(task4.getId())));
    EasyMock.expect(taskStorage.getStatus(task5.getId())).andReturn(Optional.absent());
    EasyMock.expect(taskStorage.getStatus(task6.getId())).andReturn(Optional.absent());
    EasyMock.replay(taskStorage);
    List<Object> listenerNotificationsAccumulator = new ArrayList<>();
    HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(taskStorage, listenerNotificationsAccumulator);
    Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY);
    WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class);
    EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes();
    workerHolder.setLastCompletedTaskTime(EasyMock.anyObject());
    workerHolder.resetContinuouslyFailedTasksCount();
    EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0);
    workerHolder.shutdownTask(task3.getId());
    workerHolder.shutdownTask(task5.getId());
    EasyMock.replay(workerHolder);
    Assert.assertEquals(0, taskRunner.getKnownTasks().size());
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("worker", 1, 2)), workerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task2, TaskStatus.success(task2.getId()), TaskLocation.create("worker", 3, 4)), workerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task3, TaskStatus.running(task3.getId()), TaskLocation.create("worker", 5, 6)), workerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task4, TaskStatus.success(task4.getId()), TaskLocation.create("worker", 7, 8)), workerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task5, TaskStatus.running(task5.getId()), TaskLocation.create("worker", 9, 10)), workerHolder);
    taskRunner.taskAddedOrUpdated(TaskAnnouncement.create(task6, TaskStatus.success(task6.getId()), TaskLocation.create("worker", 11, 12)), workerHolder);
    EasyMock.verify(workerHolder, taskStorage);
    Assert.assertEquals(listenerNotificationsAccumulator, ImmutableList.of(ImmutableList.of(task1.getId(), TaskLocation.create("worker", 1, 2)), ImmutableList.of(task2.getId(), TaskLocation.create("worker", 3, 4)), ImmutableList.of(task2.getId(), TaskStatus.success(task2.getId()))));
}

Example 38 with Worker

use of org.apache.druid.indexing.worker.Worker in project druid by druid-io.

the class TaskRunnerUtilsTest method testMakeWorkerURL.

@Test
public void testMakeWorkerURL() {
    final URL url = TaskRunnerUtils.makeWorkerURL(new Worker("https", "1.2.3.4:8290", "1.2.3.4", 1, "0", WorkerConfig.DEFAULT_CATEGORY), "/druid/worker/v1/task/%s/log", "foo bar&");
    Assert.assertEquals("https://1.2.3.4:8290/druid/worker/v1/task/foo%20bar%26/log", url.toString());
    Assert.assertEquals("1.2.3.4:8290", url.getAuthority());
    Assert.assertEquals("/druid/worker/v1/task/foo%20bar%26/log", url.getPath());
}

Example 39 with Worker

use of org.apache.druid.indexing.worker.Worker in project druid by druid-io.

the class PendingTaskBasedProvisioningStrategyTest method testGetExpectedWorkerCapacityWithSingleWorker.

@Test
public void testGetExpectedWorkerCapacityWithSingleWorker() {
    int workerCapacity = 3;
    Collection<ImmutableWorkerInfo> workerInfoCollection = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "localhost0", "localhost0", workerCapacity, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc()));
    int expectedWorkerCapacity = strategy.getExpectedWorkerCapacity(workerInfoCollection);
    Assert.assertEquals(workerCapacity, expectedWorkerCapacity);
}

Example 40 with Worker

use of org.apache.druid.indexing.worker.Worker in project druid by druid-io.

the class FillCapacityWithAffinityWorkerSelectStrategyTest method testIsolation.

@Test
public void testIsolation() {
    FillCapacityWorkerSelectStrategy strategy = new FillCapacityWithAffinityWorkerSelectStrategy(new AffinityConfig(ImmutableMap.of("foo", ImmutableSet.of("localhost")), false));
    ImmutableWorkerInfo worker = strategy.findWorkerForTask(new RemoteTaskRunnerConfig(), ImmutableMap.of("localhost", new ImmutableWorkerInfo(new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc())), new NoopTask(null, null, null, 1, 0, null, null, null));
    Assert.assertNull(worker);
}