Examples with CoordinatorCustomDutyGroups org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups used on opensource projects

Example 6 with CoordinatorCustomDutyGroups

use of org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups in project druid by apache.

the class DruidCoordinatorTest method testBalancerThreadNumber.

@Test
public void testBalancerThreadNumber() {
    CoordinatorDynamicConfig dynamicConfig = EasyMock.createNiceMock(CoordinatorDynamicConfig.class);
    EasyMock.expect(dynamicConfig.getBalancerComputeThreads()).andReturn(5).times(2);
    EasyMock.expect(dynamicConfig.getBalancerComputeThreads()).andReturn(10).once();
    JacksonConfigManager configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(dynamicConfig)).anyTimes();
    ScheduledExecutorFactory scheduledExecutorFactory = EasyMock.createNiceMock(ScheduledExecutorFactory.class);
    EasyMock.replay(configManager, dynamicConfig, scheduledExecutorFactory);
    DruidCoordinator c = new DruidCoordinator(druidCoordinatorConfig, null, configManager, null, null, null, () -> null, null, scheduledExecutorFactory, null, null, null, null, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), null, null, null, null, ZkEnablementConfig.ENABLED);
    DruidCoordinator.DutiesRunnable duty = c.new DutiesRunnable(Collections.emptyList(), 0, "TEST");
    // before initialization
    Assert.assertEquals(0, c.getCachedBalancerThreadNumber());
    Assert.assertNull(c.getBalancerExec());
    // first initialization
    duty.initBalancerExecutor();
    System.out.println("c.getCachedBalancerThreadNumber(): " + c.getCachedBalancerThreadNumber());
    Assert.assertEquals(5, c.getCachedBalancerThreadNumber());
    ListeningExecutorService firstExec = c.getBalancerExec();
    Assert.assertNotNull(firstExec);
    // second initialization, expect no changes as cachedBalancerThreadNumber is not changed
    duty.initBalancerExecutor();
    Assert.assertEquals(5, c.getCachedBalancerThreadNumber());
    ListeningExecutorService secondExec = c.getBalancerExec();
    Assert.assertNotNull(secondExec);
    Assert.assertTrue(firstExec == secondExec);
    // third initialization, expect executor recreated as cachedBalancerThreadNumber is changed to 10
    duty.initBalancerExecutor();
    Assert.assertEquals(10, c.getCachedBalancerThreadNumber());
    ListeningExecutorService thirdExec = c.getBalancerExec();
    Assert.assertNotNull(thirdExec);
    Assert.assertFalse(secondExec == thirdExec);
    Assert.assertFalse(firstExec == thirdExec);
}

Example 7 with CoordinatorCustomDutyGroups

use of org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups in project druid by apache.

the class CuratorDruidCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    dataSourcesSnapshot = EasyMock.createNiceMock(DataSourcesSnapshot.class);
    coordinatorRuntimeParams = EasyMock.createNiceMock(DruidCoordinatorRuntimeParams.class);
    metadataRuleManager = EasyMock.createNiceMock(MetadataRuleManager.class);
    configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference<>(CoordinatorDynamicConfig.builder().build())).anyTimes();
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorCompactionConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorCompactionConfig.empty())).anyTimes();
    EasyMock.replay(configManager);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(SEGPATH);
    curator.create().creatingParentsIfNeeded().forPath(SOURCE_LOAD_PATH);
    curator.create().creatingParentsIfNeeded().forPath(DESTINATION_LOAD_PATH);
    objectMapper = new DefaultObjectMapper();
    druidCoordinatorConfig = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"));
    sourceLoadQueueChildrenCache = new PathChildrenCache(curator, SOURCE_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_src-%d"));
    destinationLoadQueueChildrenCache = new PathChildrenCache(curator, DESTINATION_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_dest-%d"));
    sourceLoadQueuePeon = new CuratorLoadQueuePeon(curator, SOURCE_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    destinationLoadQueuePeon = new CuratorLoadQueuePeon(curator, DESTINATION_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    loadManagementPeons = new ConcurrentHashMap<>();
    scheduledExecutorFactory = (corePoolSize, nameFormat) -> Executors.newSingleThreadScheduledExecutor();
    leaderAnnouncerLatch = new CountDownLatch(1);
    leaderUnannouncerLatch = new CountDownLatch(1);
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, baseView, metadataRuleManager, () -> curator, new NoopServiceEmitter(), scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}

Example 8 with CoordinatorCustomDutyGroups

use of org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups in project druid by apache.

the class CuratorDruidCoordinatorTest method setupView.

private void setupView() throws Exception {
    baseView = new BatchServerInventoryView(zkPathsConfig, curator, jsonMapper, Predicates.alwaysTrue(), "test") {

        @Override
        public void registerSegmentCallback(Executor exec, final SegmentCallback callback) {
            super.registerSegmentCallback(exec, new SegmentCallback() {

                @Override
                public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) {
                    CallbackAction res = callback.segmentAdded(server, segment);
                    segmentAddedLatch.countDown();
                    return res;
                }

                @Override
                public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) {
                    CallbackAction res = callback.segmentRemoved(server, segment);
                    segmentRemovedLatch.countDown();
                    return res;
                }

                @Override
                public CallbackAction segmentViewInitialized() {
                    CallbackAction res = callback.segmentViewInitialized();
                    segmentViewInitLatch.countDown();
                    return res;
                }
            });
        }
    };
    serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig());
    baseView.start();
    sourceLoadQueuePeon.start();
    destinationLoadQueuePeon.start();
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, baseView, metadataRuleManager, () -> curator, new NoopServiceEmitter(), scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}

Example 9 with CoordinatorCustomDutyGroups

use of org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups in project druid by druid-io.

the class DruidCoordinatorTest method testBalancerThreadNumber.

@Test
public void testBalancerThreadNumber() {
    CoordinatorDynamicConfig dynamicConfig = EasyMock.createNiceMock(CoordinatorDynamicConfig.class);
    EasyMock.expect(dynamicConfig.getBalancerComputeThreads()).andReturn(5).times(2);
    EasyMock.expect(dynamicConfig.getBalancerComputeThreads()).andReturn(10).once();
    JacksonConfigManager configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(dynamicConfig)).anyTimes();
    ScheduledExecutorFactory scheduledExecutorFactory = EasyMock.createNiceMock(ScheduledExecutorFactory.class);
    EasyMock.replay(configManager, dynamicConfig, scheduledExecutorFactory);
    DruidCoordinator c = new DruidCoordinator(druidCoordinatorConfig, null, configManager, null, null, null, () -> null, null, scheduledExecutorFactory, null, null, null, null, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), null, null, null, null, ZkEnablementConfig.ENABLED);
    DruidCoordinator.DutiesRunnable duty = c.new DutiesRunnable(Collections.emptyList(), 0, "TEST");
    // before initialization
    Assert.assertEquals(0, c.getCachedBalancerThreadNumber());
    Assert.assertNull(c.getBalancerExec());
    // first initialization
    duty.initBalancerExecutor();
    System.out.println("c.getCachedBalancerThreadNumber(): " + c.getCachedBalancerThreadNumber());
    Assert.assertEquals(5, c.getCachedBalancerThreadNumber());
    ListeningExecutorService firstExec = c.getBalancerExec();
    Assert.assertNotNull(firstExec);
    // second initialization, expect no changes as cachedBalancerThreadNumber is not changed
    duty.initBalancerExecutor();
    Assert.assertEquals(5, c.getCachedBalancerThreadNumber());
    ListeningExecutorService secondExec = c.getBalancerExec();
    Assert.assertNotNull(secondExec);
    Assert.assertTrue(firstExec == secondExec);
    // third initialization, expect executor recreated as cachedBalancerThreadNumber is changed to 10
    duty.initBalancerExecutor();
    Assert.assertEquals(10, c.getCachedBalancerThreadNumber());
    ListeningExecutorService thirdExec = c.getBalancerExec();
    Assert.assertNotNull(thirdExec);
    Assert.assertFalse(secondExec == thirdExec);
    Assert.assertFalse(firstExec == thirdExec);
}

Example 10 with CoordinatorCustomDutyGroups

use of org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups in project druid by druid-io.

the class DruidCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    druidServer = EasyMock.createMock(DruidServer.class);
    serverInventoryView = EasyMock.createMock(BatchServerInventoryView.class);
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    dataSourcesSnapshot = EasyMock.createNiceMock(DataSourcesSnapshot.class);
    coordinatorRuntimeParams = EasyMock.createNiceMock(DruidCoordinatorRuntimeParams.class);
    metadataRuleManager = EasyMock.createNiceMock(MetadataRuleManager.class);
    JacksonConfigManager configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorDynamicConfig.builder().build())).anyTimes();
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorCompactionConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorCompactionConfig.empty())).anyTimes();
    EasyMock.replay(configManager);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(LOADPATH);
    objectMapper = new DefaultObjectMapper();
    druidCoordinatorConfig = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"));
    pathChildrenCache = new PathChildrenCache(curator, LOADPATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache-%d"));
    loadQueuePeon = new CuratorLoadQueuePeon(curator, LOADPATH, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon-%d"), druidCoordinatorConfig);
    loadQueuePeon.start();
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    loadManagementPeons = new ConcurrentHashMap<>();
    scheduledExecutorFactory = new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int corePoolSize, final String nameFormat) {
            return Executors.newSingleThreadScheduledExecutor();
        }
    };
    leaderAnnouncerLatch = new CountDownLatch(1);
    leaderUnannouncerLatch = new CountDownLatch(1);
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, new HashSet<>(), new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}