Search in sources :

Example 21 with ZkPathsConfig

use of org.apache.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class CliCoordinator method getModules.

@Override
protected List<? extends Module> getModules() {
    List<Module> modules = new ArrayList<>();
    modules.add(JettyHttpClientModule.global());
    modules.add(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to(TieredBrokerConfig.DEFAULT_COORDINATOR_SERVICE_NAME);
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
            binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(8281);
            ConfigProvider.bind(binder, DruidCoordinatorConfig.class);
            binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class);
            JsonConfigProvider.bind(binder, "druid.manager.segments", SegmentsMetadataManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.balancer", BalancerStrategyFactory.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.segment", CoordinatorSegmentWatcherConfig.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class);
            binder.bind(RedirectFilter.class).in(LazySingleton.class);
            if (beOverlord) {
                binder.bind(RedirectInfo.class).to(CoordinatorOverlordRedirectInfo.class).in(LazySingleton.class);
            } else {
                binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class);
            }
            binder.bind(SegmentsMetadataManager.class).toProvider(SegmentsMetadataManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(MetadataRuleManager.class).toProvider(MetadataRuleManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(AuditManager.class).toProvider(AuditManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(IndexingServiceClient.class).to(HttpIndexingServiceClient.class).in(LazySingleton.class);
            binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class);
            binder.bind(CoordinatorServerView.class);
            binder.bind(DruidCoordinator.class);
            LifecycleModule.register(binder, CoordinatorServerView.class);
            LifecycleModule.register(binder, MetadataStorage.class);
            LifecycleModule.register(binder, DruidCoordinator.class);
            binder.bind(JettyServerInitializer.class).to(CoordinatorJettyServerInitializer.class);
            Jerseys.addResource(binder, CoordinatorResource.class);
            Jerseys.addResource(binder, CompactionResource.class);
            Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
            Jerseys.addResource(binder, CoordinatorCompactionConfigsResource.class);
            Jerseys.addResource(binder, TiersResource.class);
            Jerseys.addResource(binder, RulesResource.class);
            Jerseys.addResource(binder, ServersResource.class);
            Jerseys.addResource(binder, DataSourcesResource.class);
            Jerseys.addResource(binder, MetadataResource.class);
            Jerseys.addResource(binder, IntervalsResource.class);
            Jerseys.addResource(binder, LookupCoordinatorResource.class);
            Jerseys.addResource(binder, ClusterResource.class);
            Jerseys.addResource(binder, HttpServerInventoryViewResource.class);
            LifecycleModule.register(binder, Server.class);
            LifecycleModule.register(binder, DataSourcesResource.class);
            // Binding for Set of indexing service coordinator Duty
            final ConditionalMultibind<CoordinatorDuty> conditionalIndexingServiceDutyMultibind = ConditionalMultibind.create(properties, binder, CoordinatorDuty.class, CoordinatorIndexingServiceDuty.class);
            if (conditionalIndexingServiceDutyMultibind.matchCondition("druid.coordinator.merge.on", Predicates.equalTo("true"))) {
                throw new UnsupportedOperationException("'druid.coordinator.merge.on' is not supported anymore. " + "Please consider using Coordinator's automatic compaction instead. " + "See https://druid.apache.org/docs/latest/operations/segment-optimization.html and " + "https://druid.apache.org/docs/latest/operations/api-reference.html#compaction-configuration " + "for more details about compaction.");
            }
            conditionalIndexingServiceDutyMultibind.addConditionBinding("druid.coordinator.kill.on", "true", Predicates.equalTo("true"), KillUnusedSegments.class);
            conditionalIndexingServiceDutyMultibind.addConditionBinding("druid.coordinator.kill.pendingSegments.on", "true", Predicates.equalTo("true"), KillStalePendingSegments.class);
            // Binding for Set of metadata store management coordinator Ddty
            final ConditionalMultibind<CoordinatorDuty> conditionalMetadataStoreManagementDutyMultibind = ConditionalMultibind.create(properties, binder, CoordinatorDuty.class, CoordinatorMetadataStoreManagementDuty.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.supervisor.on", "true", Predicates.equalTo("true"), KillSupervisors.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.audit.on", "true", Predicates.equalTo("true"), KillAuditLog.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.rule.on", "true", Predicates.equalTo("true"), KillRules.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.datasource.on", "true", Predicates.equalTo("true"), KillDatasourceMetadata.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.compaction.on", Predicates.equalTo("true"), KillCompactionConfig.class);
            bindAnnouncer(binder, Coordinator.class, DiscoverySideEffectsProvider.create());
            Jerseys.addResource(binder, SelfDiscoveryResource.class);
            LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class));
            if (!beOverlord) {
                // These are needed to deserialize SupervisorSpec for Supervisor Auto Cleanup
                binder.bind(TaskStorage.class).toProvider(Providers.of(null));
                binder.bind(TaskMaster.class).toProvider(Providers.of(null));
                binder.bind(RowIngestionMetersFactory.class).toProvider(Providers.of(null));
            }
            binder.bind(CoordinatorCustomDutyGroups.class).toProvider(new CoordinatorCustomDutyGroupsProvider()).in(LazySingleton.class);
        }

        @Provides
        @LazySingleton
        public LoadQueueTaskMaster getLoadQueueTaskMaster(Provider<CuratorFramework> curatorFrameworkProvider, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidCoordinatorConfig config, @EscalatedGlobal HttpClient httpClient, ZkPathsConfig zkPaths, Lifecycle lifecycle) {
            boolean useHttpLoadQueuePeon = "http".equalsIgnoreCase(config.getLoadQueuePeonType());
            ExecutorService callBackExec;
            if (useHttpLoadQueuePeon) {
                callBackExec = Execs.singleThreaded("LoadQueuePeon-callbackexec--%d");
            } else {
                callBackExec = Execs.multiThreaded(config.getNumCuratorCallBackThreads(), "LoadQueuePeon-callbackexec--%d");
            }
            ExecutorServices.manageLifecycle(lifecycle, callBackExec);
            return new LoadQueueTaskMaster(curatorFrameworkProvider, jsonMapper, factory.create(1, "Master-PeonExec--%d"), callBackExec, config, httpClient, zkPaths);
        }
    });
    if (beOverlord) {
        modules.addAll(new CliOverlord().getModules(false));
    } else {
        // Only add LookupSerdeModule if !beOverlord, since CliOverlord includes it, and having two copies causes
        // the injector to get confused due to having multiple bindings for the same classes.
        modules.add(new LookupSerdeModule());
    }
    return modules;
}
Also used : LookupSerdeModule(org.apache.druid.query.lookup.LookupSerdeModule) Server(org.eclipse.jetty.server.Server) ArrayList(java.util.ArrayList) LookupCoordinatorResource(org.apache.druid.server.http.LookupCoordinatorResource) CoordinatorResource(org.apache.druid.server.http.CoordinatorResource) ServersResource(org.apache.druid.server.http.ServersResource) LazySingleton(org.apache.druid.guice.LazySingleton) SegmentsMetadataManagerConfig(org.apache.druid.metadata.SegmentsMetadataManagerConfig) CuratorFramework(org.apache.curator.framework.CuratorFramework) CachingCostBalancerStrategyConfig(org.apache.druid.server.coordinator.CachingCostBalancerStrategyConfig) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) KillRules(org.apache.druid.server.coordinator.duty.KillRules) AuditManager(org.apache.druid.audit.AuditManager) DruidCoordinator(org.apache.druid.server.coordinator.DruidCoordinator) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) MetadataStorageProvider(org.apache.druid.metadata.MetadataStorageProvider) MetadataResource(org.apache.druid.server.http.MetadataResource) CoordinatorMetadataStoreManagementDuty(org.apache.druid.guice.annotations.CoordinatorMetadataStoreManagementDuty) IndexingServiceClient(org.apache.druid.client.indexing.IndexingServiceClient) HttpIndexingServiceClient(org.apache.druid.client.indexing.HttpIndexingServiceClient) KillCompactionConfig(org.apache.druid.server.coordinator.duty.KillCompactionConfig) ManageLifecycle(org.apache.druid.guice.ManageLifecycle) Lifecycle(org.apache.druid.java.util.common.lifecycle.Lifecycle) CoordinatorSegmentWatcherConfig(org.apache.druid.client.CoordinatorSegmentWatcherConfig) LookupCoordinatorResource(org.apache.druid.server.http.LookupCoordinatorResource) HttpServerInventoryViewResource(org.apache.druid.client.HttpServerInventoryViewResource) ScheduledExecutorFactory(org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory) MetadataStorage(org.apache.druid.metadata.MetadataStorage) RedirectInfo(org.apache.druid.server.http.RedirectInfo) CoordinatorRedirectInfo(org.apache.druid.server.http.CoordinatorRedirectInfo) LifecycleModule(org.apache.druid.guice.LifecycleModule) JettyHttpClientModule(org.apache.druid.guice.http.JettyHttpClientModule) Module(com.google.inject.Module) LookupSerdeModule(org.apache.druid.query.lookup.LookupSerdeModule) KillAuditLog(org.apache.druid.server.coordinator.duty.KillAuditLog) LoadQueueTaskMaster(org.apache.druid.server.coordinator.LoadQueueTaskMaster) SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) KillStalePendingSegments(org.apache.druid.server.coordinator.KillStalePendingSegments) ClusterResource(org.apache.druid.server.http.ClusterResource) Binder(com.google.inject.Binder) TiersResource(org.apache.druid.server.http.TiersResource) CoordinatorDuty(org.apache.druid.server.coordinator.duty.CoordinatorDuty) KillUnusedSegments(org.apache.druid.server.coordinator.duty.KillUnusedSegments) DruidCoordinatorConfig(org.apache.druid.server.coordinator.DruidCoordinatorConfig) CoordinatorCompactionConfigsResource(org.apache.druid.server.http.CoordinatorCompactionConfigsResource) LookupCoordinatorManagerConfig(org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) KillDatasourceMetadata(org.apache.druid.server.coordinator.duty.KillDatasourceMetadata) RulesResource(org.apache.druid.server.http.RulesResource) CompactionResource(org.apache.druid.server.http.CompactionResource) CoordinatorIndexingServiceDuty(org.apache.druid.guice.annotations.CoordinatorIndexingServiceDuty) BalancerStrategyFactory(org.apache.druid.server.coordinator.BalancerStrategyFactory) SelfDiscoveryResource(org.apache.druid.server.http.SelfDiscoveryResource) ManageLifecycle(org.apache.druid.guice.ManageLifecycle) DruidCoordinator(org.apache.druid.server.coordinator.DruidCoordinator) Coordinator(org.apache.druid.client.coordinator.Coordinator) Provides(com.google.inject.Provides) CoordinatorDynamicConfigsResource(org.apache.druid.server.http.CoordinatorDynamicConfigsResource) IntervalsResource(org.apache.druid.server.http.IntervalsResource) MetadataRuleManagerConfig(org.apache.druid.metadata.MetadataRuleManagerConfig) HttpClient(org.apache.druid.java.util.http.client.HttpClient) ExecutorService(java.util.concurrent.ExecutorService) KillSupervisors(org.apache.druid.server.coordinator.duty.KillSupervisors) DataSourcesResource(org.apache.druid.server.http.DataSourcesResource) CoordinatorServerView(org.apache.druid.client.CoordinatorServerView) ConditionalMultibind(org.apache.druid.guice.ConditionalMultibind)

Example 22 with ZkPathsConfig

use of org.apache.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class DruidCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    druidServer = EasyMock.createMock(DruidServer.class);
    serverInventoryView = EasyMock.createMock(BatchServerInventoryView.class);
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    dataSourcesSnapshot = EasyMock.createNiceMock(DataSourcesSnapshot.class);
    coordinatorRuntimeParams = EasyMock.createNiceMock(DruidCoordinatorRuntimeParams.class);
    metadataRuleManager = EasyMock.createNiceMock(MetadataRuleManager.class);
    JacksonConfigManager configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorDynamicConfig.builder().build())).anyTimes();
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorCompactionConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorCompactionConfig.empty())).anyTimes();
    EasyMock.replay(configManager);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(LOADPATH);
    objectMapper = new DefaultObjectMapper();
    druidCoordinatorConfig = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"));
    pathChildrenCache = new PathChildrenCache(curator, LOADPATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache-%d"));
    loadQueuePeon = new CuratorLoadQueuePeon(curator, LOADPATH, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon-%d"), druidCoordinatorConfig);
    loadQueuePeon.start();
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    loadManagementPeons = new ConcurrentHashMap<>();
    scheduledExecutorFactory = new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int corePoolSize, final String nameFormat) {
            return Executors.newSingleThreadScheduledExecutor();
        }
    };
    leaderAnnouncerLatch = new CountDownLatch(1);
    leaderUnannouncerLatch = new CountDownLatch(1);
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, new HashSet<>(), new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}
Also used : SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) HashSet(java.util.HashSet) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) DruidServer(org.apache.druid.client.DruidServer) ImmutableDruidServer(org.apache.druid.client.ImmutableDruidServer) AtomicReference(java.util.concurrent.atomic.AtomicReference) Duration(org.joda.time.Duration) CountDownLatch(java.util.concurrent.CountDownLatch) ScheduledExecutorFactory(org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory) PathChildrenCache(org.apache.curator.framework.recipes.cache.PathChildrenCache) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) DruidNode(org.apache.druid.server.DruidNode) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) Before(org.junit.Before)

Example 23 with ZkPathsConfig

use of org.apache.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class DruidCoordinatorTest method testCompactSegmentsDutyWhenCustomDutyGroupEmpty.

@Test
public void testCompactSegmentsDutyWhenCustomDutyGroupEmpty() {
    CoordinatorCustomDutyGroups emptyCustomDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of());
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), emptyCustomDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().anyMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should not exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
    // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}
Also used : LookupCoordinatorManager(org.apache.druid.server.lookup.cache.LookupCoordinatorManager) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) DruidServer(org.apache.druid.client.DruidServer) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) ForeverBroadcastDistributionRule(org.apache.druid.server.coordinator.rules.ForeverBroadcastDistributionRule) Event(org.apache.druid.java.util.emitter.core.Event) After(org.junit.After) Map(java.util.Map) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) ServerType(org.apache.druid.server.coordination.ServerType) ForeverLoadRule(org.apache.druid.server.coordinator.rules.ForeverLoadRule) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) ImmutableSet(com.google.common.collect.ImmutableSet) Execs(org.apache.druid.java.util.common.concurrent.Execs) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CoordinatorCustomDutyGroup(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroup) Rule(org.apache.druid.server.coordinator.rules.Rule) CuratorUtils(org.apache.druid.curator.CuratorUtils) Executors(java.util.concurrent.Executors) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) KillSupervisorsCustomDuty(org.apache.druid.server.coordinator.duty.KillSupervisorsCustomDuty) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) PathChildrenCacheEvent(org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent) DruidLeaderSelector(org.apache.druid.discovery.DruidLeaderSelector) DataSegment(org.apache.druid.timeline.DataSegment) CoordinatorDuty(org.apache.druid.server.coordinator.duty.CoordinatorDuty) SegmentId(org.apache.druid.timeline.SegmentId) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) DruidDataSource(org.apache.druid.client.DruidDataSource) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) Intervals(org.apache.druid.java.util.common.Intervals) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) Duration(org.joda.time.Duration) ZkEnablementConfig(org.apache.druid.curator.ZkEnablementConfig) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) AtomicReference(java.util.concurrent.atomic.AtomicReference) ScheduledExecutorFactory(org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) Object2LongMap(it.unimi.dsi.fastutil.objects.Object2LongMap) ImmutableList(com.google.common.collect.ImmutableList) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) CompactSegments(org.apache.druid.server.coordinator.duty.CompactSegments) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) ImmutableDruidServer(org.apache.druid.client.ImmutableDruidServer) Nullable(javax.annotation.Nullable) Before(org.junit.Before) Capture(org.easymock.Capture) SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) CuratorTestBase(org.apache.druid.curator.CuratorTestBase) Object2IntMap(it.unimi.dsi.fastutil.objects.Object2IntMap) CoordinatorCustomDuty(org.apache.druid.server.coordinator.duty.CoordinatorCustomDuty) DruidNode(org.apache.druid.server.DruidNode) PathChildrenCache(org.apache.curator.framework.recipes.cache.PathChildrenCache) Assert(org.junit.Assert) Collections(java.util.Collections) BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) CompactSegments(org.apache.druid.server.coordinator.duty.CompactSegments) CoordinatorDuty(org.apache.druid.server.coordinator.duty.CoordinatorDuty) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) DruidNode(org.apache.druid.server.DruidNode) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 24 with ZkPathsConfig

use of org.apache.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class DruidCoordinatorTest method testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments.

@Test
public void testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments() {
    DruidCoordinatorConfig differentConfigUsedInCustomGroup = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"), false);
    CoordinatorCustomDutyGroup compactSegmentCustomGroup = new CoordinatorCustomDutyGroup("group1", Duration.standardSeconds(1), ImmutableList.of(new CompactSegments(differentConfigUsedInCustomGroup, null, null)));
    CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(compactSegmentCustomGroup));
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), customDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is enabled in Custom Duty Group, then CompactSegments must not be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().noneMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertFalse(compactSegmentsDutyFromCustomGroups.isEmpty());
    Assert.assertEquals(1, compactSegmentsDutyFromCustomGroups.size());
    Assert.assertNotNull(compactSegmentsDutyFromCustomGroups.get(0));
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.get(0) instanceof CompactSegments);
    // CompactSegments returned by this method should be from the Custom Duty Group
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertNotEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
    // We should get the CompactSegment from the custom duty group which was created with a different config than the config in DruidCoordinator
    Assert.assertEquals(differentConfigUsedInCustomGroup.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}
Also used : LookupCoordinatorManager(org.apache.druid.server.lookup.cache.LookupCoordinatorManager) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) DruidServer(org.apache.druid.client.DruidServer) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) ForeverBroadcastDistributionRule(org.apache.druid.server.coordinator.rules.ForeverBroadcastDistributionRule) Event(org.apache.druid.java.util.emitter.core.Event) After(org.junit.After) Map(java.util.Map) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) ServerType(org.apache.druid.server.coordination.ServerType) ForeverLoadRule(org.apache.druid.server.coordinator.rules.ForeverLoadRule) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) ImmutableSet(com.google.common.collect.ImmutableSet) Execs(org.apache.druid.java.util.common.concurrent.Execs) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CoordinatorCustomDutyGroup(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroup) Rule(org.apache.druid.server.coordinator.rules.Rule) CuratorUtils(org.apache.druid.curator.CuratorUtils) Executors(java.util.concurrent.Executors) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) KillSupervisorsCustomDuty(org.apache.druid.server.coordinator.duty.KillSupervisorsCustomDuty) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) PathChildrenCacheEvent(org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent) DruidLeaderSelector(org.apache.druid.discovery.DruidLeaderSelector) DataSegment(org.apache.druid.timeline.DataSegment) CoordinatorDuty(org.apache.druid.server.coordinator.duty.CoordinatorDuty) SegmentId(org.apache.druid.timeline.SegmentId) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) DruidDataSource(org.apache.druid.client.DruidDataSource) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) Intervals(org.apache.druid.java.util.common.Intervals) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) Duration(org.joda.time.Duration) ZkEnablementConfig(org.apache.druid.curator.ZkEnablementConfig) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) AtomicReference(java.util.concurrent.atomic.AtomicReference) ScheduledExecutorFactory(org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) Object2LongMap(it.unimi.dsi.fastutil.objects.Object2LongMap) ImmutableList(com.google.common.collect.ImmutableList) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) CompactSegments(org.apache.druid.server.coordinator.duty.CompactSegments) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) ImmutableDruidServer(org.apache.druid.client.ImmutableDruidServer) Nullable(javax.annotation.Nullable) Before(org.junit.Before) Capture(org.easymock.Capture) SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) CuratorTestBase(org.apache.druid.curator.CuratorTestBase) Object2IntMap(it.unimi.dsi.fastutil.objects.Object2IntMap) CoordinatorCustomDuty(org.apache.druid.server.coordinator.duty.CoordinatorCustomDuty) DruidNode(org.apache.druid.server.DruidNode) PathChildrenCache(org.apache.curator.framework.recipes.cache.PathChildrenCache) Assert(org.junit.Assert) Collections(java.util.Collections) BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) Duration(org.joda.time.Duration) CompactSegments(org.apache.druid.server.coordinator.duty.CompactSegments) CoordinatorDuty(org.apache.druid.server.coordinator.duty.CoordinatorDuty) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) DruidNode(org.apache.druid.server.DruidNode) CoordinatorCustomDutyGroup(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroup) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 25 with ZkPathsConfig

use of org.apache.druid.server.initialization.ZkPathsConfig in project druid by druid-io.

the class BatchDataSegmentAnnouncerTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(false)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(TEST_BASE_PATH);
    jsonMapper = TestHelper.makeJsonMapper();
    announcer = new TestAnnouncer(cf, Execs.directExecutor());
    announcer.start();
    segmentReader = new SegmentReader(cf, jsonMapper);
    skipDimensionsAndMetrics = false;
    skipLoadSpec = false;
    segmentAnnouncer = new BatchDataSegmentAnnouncer(new DruidServerMetadata("id", "host", null, Long.MAX_VALUE, ServerType.HISTORICAL, "tier", 0), new BatchDataSegmentAnnouncerConfig() {

        @Override
        public int getSegmentsPerNode() {
            return 50;
        }

        @Override
        public long getMaxBytesPerNode() {
            return maxBytesPerNode.get();
        }

        @Override
        public boolean isSkipDimensionsAndMetrics() {
            return skipDimensionsAndMetrics;
        }

        @Override
        public boolean isSkipLoadSpec() {
            return skipLoadSpec;
        }
    }, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return TEST_BASE_PATH;
        }
    }, announcer, jsonMapper);
    testSegments = new HashSet<>();
    for (int i = 0; i < 100; i++) {
        testSegments.add(makeSegment(i));
    }
    exec = Execs.multiThreaded(NUM_THREADS, "BatchDataSegmentAnnouncerTest-%d");
}
Also used : TestingCluster(org.apache.curator.test.TestingCluster) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) BatchDataSegmentAnnouncerConfig(org.apache.druid.server.initialization.BatchDataSegmentAnnouncerConfig) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) PotentiallyGzippedCompressionProvider(org.apache.druid.curator.PotentiallyGzippedCompressionProvider) BatchDataSegmentAnnouncer(org.apache.druid.server.coordination.BatchDataSegmentAnnouncer) Before(org.junit.Before)

Aggregations

ZkPathsConfig (org.apache.druid.server.initialization.ZkPathsConfig)26 AtomicReference (java.util.concurrent.atomic.AtomicReference)16 DruidNode (org.apache.druid.server.DruidNode)15 Test (org.junit.Test)15 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)14 CuratorFramework (org.apache.curator.framework.CuratorFramework)13 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)12 IndexerZkConfig (org.apache.druid.server.initialization.IndexerZkConfig)12 ImmutableList (com.google.common.collect.ImmutableList)11 List (java.util.List)11 ArrayList (java.util.ArrayList)10 HttpClient (org.apache.druid.java.util.http.client.HttpClient)10 Before (org.junit.Before)10 DruidNodeDiscoveryProvider (org.apache.druid.discovery.DruidNodeDiscoveryProvider)9 TaskRunnerListener (org.apache.druid.indexing.overlord.TaskRunnerListener)9 HttpRemoteTaskRunnerConfig (org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig)9 Worker (org.apache.druid.indexing.worker.Worker)9 DiscoveryDruidNode (org.apache.druid.discovery.DiscoveryDruidNode)8 TaskStorage (org.apache.druid.indexing.overlord.TaskStorage)8 CoordinatorCustomDutyGroups (org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups)8