Examples with CoordinatorDuty org.apache.druid.server.coordinator.duty.CoordinatorDuty used on opensource projects

Example 1 with CoordinatorDuty

use of org.apache.druid.server.coordinator.duty.CoordinatorDuty in project druid by druid-io.

the class DruidCoordinatorTest method testInitializeCompactSegmentsDutyWhenCustomDutyGroupDoesNotContainsCompactSegments.

@Test
public void testInitializeCompactSegmentsDutyWhenCustomDutyGroupDoesNotContainsCompactSegments() {
    CoordinatorCustomDutyGroup group = new CoordinatorCustomDutyGroup("group1", Duration.standardSeconds(1), ImmutableList.of(new KillSupervisorsCustomDuty(new Duration("PT1S"), null)));
    CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(group));
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), customDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().anyMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should not exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
    // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}

Example 2 with CoordinatorDuty

use of org.apache.druid.server.coordinator.duty.CoordinatorDuty in project druid by druid-io.

the class DruidCoordinator method makeIndexingServiceDuties.

@VisibleForTesting
List<CoordinatorDuty> makeIndexingServiceDuties() {
    List<CoordinatorDuty> duties = new ArrayList<>();
    duties.add(new LogUsedSegments());
    duties.addAll(indexingServiceDuties);
    // We do not have to add compactSegments if it is already enabled in the custom duty group
    if (getCompactSegmentsDutyFromCustomGroups().isEmpty()) {
        duties.addAll(makeCompactSegmentsDuty());
    }
    log.debug("Done making indexing service duties %s", duties.stream().map(duty -> duty.getClass().getName()).collect(Collectors.toList()));
    return ImmutableList.copyOf(duties);
}

Example 3 with CoordinatorDuty

use of org.apache.druid.server.coordinator.duty.CoordinatorDuty in project druid by druid-io.

the class CliCoordinator method getModules.

@Override
protected List<? extends Module> getModules() {
    List<Module> modules = new ArrayList<>();
    modules.add(JettyHttpClientModule.global());
    modules.add(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to(TieredBrokerConfig.DEFAULT_COORDINATOR_SERVICE_NAME);
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
            binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(8281);
            ConfigProvider.bind(binder, DruidCoordinatorConfig.class);
            binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class);
            JsonConfigProvider.bind(binder, "druid.manager.segments", SegmentsMetadataManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.balancer", BalancerStrategyFactory.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.segment", CoordinatorSegmentWatcherConfig.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class);
            binder.bind(RedirectFilter.class).in(LazySingleton.class);
            if (beOverlord) {
                binder.bind(RedirectInfo.class).to(CoordinatorOverlordRedirectInfo.class).in(LazySingleton.class);
            } else {
                binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class);
            }
            binder.bind(SegmentsMetadataManager.class).toProvider(SegmentsMetadataManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(MetadataRuleManager.class).toProvider(MetadataRuleManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(AuditManager.class).toProvider(AuditManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(IndexingServiceClient.class).to(HttpIndexingServiceClient.class).in(LazySingleton.class);
            binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class);
            binder.bind(CoordinatorServerView.class);
            binder.bind(DruidCoordinator.class);
            LifecycleModule.register(binder, CoordinatorServerView.class);
            LifecycleModule.register(binder, MetadataStorage.class);
            LifecycleModule.register(binder, DruidCoordinator.class);
            binder.bind(JettyServerInitializer.class).to(CoordinatorJettyServerInitializer.class);
            Jerseys.addResource(binder, CoordinatorResource.class);
            Jerseys.addResource(binder, CompactionResource.class);
            Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
            Jerseys.addResource(binder, CoordinatorCompactionConfigsResource.class);
            Jerseys.addResource(binder, TiersResource.class);
            Jerseys.addResource(binder, RulesResource.class);
            Jerseys.addResource(binder, ServersResource.class);
            Jerseys.addResource(binder, DataSourcesResource.class);
            Jerseys.addResource(binder, MetadataResource.class);
            Jerseys.addResource(binder, IntervalsResource.class);
            Jerseys.addResource(binder, LookupCoordinatorResource.class);
            Jerseys.addResource(binder, ClusterResource.class);
            Jerseys.addResource(binder, HttpServerInventoryViewResource.class);
            LifecycleModule.register(binder, Server.class);
            LifecycleModule.register(binder, DataSourcesResource.class);
            // Binding for Set of indexing service coordinator Duty
            final ConditionalMultibind<CoordinatorDuty> conditionalIndexingServiceDutyMultibind = ConditionalMultibind.create(properties, binder, CoordinatorDuty.class, CoordinatorIndexingServiceDuty.class);
            if (conditionalIndexingServiceDutyMultibind.matchCondition("druid.coordinator.merge.on", Predicates.equalTo("true"))) {
                throw new UnsupportedOperationException("'druid.coordinator.merge.on' is not supported anymore. " + "Please consider using Coordinator's automatic compaction instead. " + "See https://druid.apache.org/docs/latest/operations/segment-optimization.html and " + "https://druid.apache.org/docs/latest/operations/api-reference.html#compaction-configuration " + "for more details about compaction.");
            }
            conditionalIndexingServiceDutyMultibind.addConditionBinding("druid.coordinator.kill.on", "true", Predicates.equalTo("true"), KillUnusedSegments.class);
            conditionalIndexingServiceDutyMultibind.addConditionBinding("druid.coordinator.kill.pendingSegments.on", "true", Predicates.equalTo("true"), KillStalePendingSegments.class);
            // Binding for Set of metadata store management coordinator Ddty
            final ConditionalMultibind<CoordinatorDuty> conditionalMetadataStoreManagementDutyMultibind = ConditionalMultibind.create(properties, binder, CoordinatorDuty.class, CoordinatorMetadataStoreManagementDuty.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.supervisor.on", "true", Predicates.equalTo("true"), KillSupervisors.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.audit.on", "true", Predicates.equalTo("true"), KillAuditLog.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.rule.on", "true", Predicates.equalTo("true"), KillRules.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.datasource.on", "true", Predicates.equalTo("true"), KillDatasourceMetadata.class);
            conditionalMetadataStoreManagementDutyMultibind.addConditionBinding("druid.coordinator.kill.compaction.on", Predicates.equalTo("true"), KillCompactionConfig.class);
            bindAnnouncer(binder, Coordinator.class, DiscoverySideEffectsProvider.create());
            Jerseys.addResource(binder, SelfDiscoveryResource.class);
            LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class));
            if (!beOverlord) {
                // These are needed to deserialize SupervisorSpec for Supervisor Auto Cleanup
                binder.bind(TaskStorage.class).toProvider(Providers.of(null));
                binder.bind(TaskMaster.class).toProvider(Providers.of(null));
                binder.bind(RowIngestionMetersFactory.class).toProvider(Providers.of(null));
            }
            binder.bind(CoordinatorCustomDutyGroups.class).toProvider(new CoordinatorCustomDutyGroupsProvider()).in(LazySingleton.class);
        }

        @Provides
        @LazySingleton
        public LoadQueueTaskMaster getLoadQueueTaskMaster(Provider<CuratorFramework> curatorFrameworkProvider, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidCoordinatorConfig config, @EscalatedGlobal HttpClient httpClient, ZkPathsConfig zkPaths, Lifecycle lifecycle) {
            boolean useHttpLoadQueuePeon = "http".equalsIgnoreCase(config.getLoadQueuePeonType());
            ExecutorService callBackExec;
            if (useHttpLoadQueuePeon) {
                callBackExec = Execs.singleThreaded("LoadQueuePeon-callbackexec--%d");
            } else {
                callBackExec = Execs.multiThreaded(config.getNumCuratorCallBackThreads(), "LoadQueuePeon-callbackexec--%d");
            }
            ExecutorServices.manageLifecycle(lifecycle, callBackExec);
            return new LoadQueueTaskMaster(curatorFrameworkProvider, jsonMapper, factory.create(1, "Master-PeonExec--%d"), callBackExec, config, httpClient, zkPaths);
        }
    });
    if (beOverlord) {
        modules.addAll(new CliOverlord().getModules(false));
    } else {
        // Only add LookupSerdeModule if !beOverlord, since CliOverlord includes it, and having two copies causes
        // the injector to get confused due to having multiple bindings for the same classes.
        modules.add(new LookupSerdeModule());
    }
    return modules;
}

Example 4 with CoordinatorDuty

use of org.apache.druid.server.coordinator.duty.CoordinatorDuty in project druid by druid-io.

the class DruidCoordinatorTest method testCompactSegmentsDutyWhenCustomDutyGroupEmpty.

@Test
public void testCompactSegmentsDutyWhenCustomDutyGroupEmpty() {
    CoordinatorCustomDutyGroups emptyCustomDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of());
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), emptyCustomDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().anyMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should not exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
    // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}

Example 5 with CoordinatorDuty

use of org.apache.druid.server.coordinator.duty.CoordinatorDuty in project druid by druid-io.

the class DruidCoordinatorTest method testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments.

@Test
public void testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments() {
    DruidCoordinatorConfig differentConfigUsedInCustomGroup = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"), false);
    CoordinatorCustomDutyGroup compactSegmentCustomGroup = new CoordinatorCustomDutyGroup("group1", Duration.standardSeconds(1), ImmutableList.of(new CompactSegments(differentConfigUsedInCustomGroup, null, null)));
    CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(compactSegmentCustomGroup));
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), customDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is enabled in Custom Duty Group, then CompactSegments must not be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().noneMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertFalse(compactSegmentsDutyFromCustomGroups.isEmpty());
    Assert.assertEquals(1, compactSegmentsDutyFromCustomGroups.size());
    Assert.assertNotNull(compactSegmentsDutyFromCustomGroups.get(0));
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.get(0) instanceof CompactSegments);
    // CompactSegments returned by this method should be from the Custom Duty Group
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertNotEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
    // We should get the CompactSegment from the custom duty group which was created with a different config than the config in DruidCoordinator
    Assert.assertEquals(differentConfigUsedInCustomGroup.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}