Examples with ScheduledExecutorFactory org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory used on opensource projects

Example 6 with ScheduledExecutorFactory

use of org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory in project druid by druid-io.

the class DruidCoordinatorTest method testCompactSegmentsDutyWhenCustomDutyGroupEmpty.

@Test
public void testCompactSegmentsDutyWhenCustomDutyGroupEmpty() {
    CoordinatorCustomDutyGroups emptyCustomDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of());
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), emptyCustomDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is not enabled in Custom Duty Group, then CompactSegments must be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().anyMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should not exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.isEmpty());
    // CompactSegments returned by this method should be created using the DruidCoordinatorConfig in the DruidCoordinator
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}

Example 7 with ScheduledExecutorFactory

use of org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory in project druid by druid-io.

the class DruidCoordinatorTest method testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments.

@Test
public void testInitializeCompactSegmentsDutyWhenCustomDutyGroupContainsCompactSegments() {
    DruidCoordinatorConfig differentConfigUsedInCustomGroup = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"), false);
    CoordinatorCustomDutyGroup compactSegmentCustomGroup = new CoordinatorCustomDutyGroup("group1", Duration.standardSeconds(1), ImmutableList.of(new CompactSegments(differentConfigUsedInCustomGroup, null, null)));
    CoordinatorCustomDutyGroups customDutyGroups = new CoordinatorCustomDutyGroups(ImmutableSet.of(compactSegmentCustomGroup));
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, null, segmentsMetadataManager, serverInventoryView, metadataRuleManager, () -> curator, serviceEmitter, scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, ImmutableSet.of(), new HashSet<>(), customDutyGroups, new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
    // Since CompactSegments is enabled in Custom Duty Group, then CompactSegments must not be created in IndexingServiceDuties
    List<CoordinatorDuty> indexingDuties = coordinator.makeIndexingServiceDuties();
    Assert.assertTrue(indexingDuties.stream().noneMatch(coordinatorDuty -> coordinatorDuty instanceof CompactSegments));
    // CompactSegments should exist in Custom Duty Group
    List<CompactSegments> compactSegmentsDutyFromCustomGroups = coordinator.getCompactSegmentsDutyFromCustomGroups();
    Assert.assertFalse(compactSegmentsDutyFromCustomGroups.isEmpty());
    Assert.assertEquals(1, compactSegmentsDutyFromCustomGroups.size());
    Assert.assertNotNull(compactSegmentsDutyFromCustomGroups.get(0));
    Assert.assertTrue(compactSegmentsDutyFromCustomGroups.get(0) instanceof CompactSegments);
    // CompactSegments returned by this method should be from the Custom Duty Group
    CompactSegments duty = coordinator.initializeCompactSegmentsDuty();
    Assert.assertNotNull(duty);
    Assert.assertNotEquals(druidCoordinatorConfig.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
    // We should get the CompactSegment from the custom duty group which was created with a different config than the config in DruidCoordinator
    Assert.assertEquals(differentConfigUsedInCustomGroup.getCompactionSkipLockedIntervals(), duty.isSkipLockedIntervals());
}

Example 8 with ScheduledExecutorFactory

use of org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory in project druid by druid-io.

the class SegmentLoadDropHandlerTest method setUp.

@Before
public void setUp() throws IOException {
    try {
        testStorageLocation = new TestStorageLocation(temporaryFolder);
        infoDir = testStorageLocation.getInfoDir();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    locations = Collections.singletonList(testStorageLocation.toStorageLocationConfig());
    scheduledRunnable = new ArrayList<>();
    segmentCacheManager = new CacheTestSegmentCacheManager();
    segmentLoader = new CacheTestSegmentLoader();
    segmentManager = new SegmentManager(segmentLoader);
    segmentsAnnouncedByMe = new ConcurrentSkipListSet<>();
    announceCount = new AtomicInteger(0);
    announcer = new DataSegmentAnnouncer() {

        @Override
        public void announceSegment(DataSegment segment) {
            segmentsAnnouncedByMe.add(segment);
            announceCount.incrementAndGet();
        }

        @Override
        public void unannounceSegment(DataSegment segment) {
            segmentsAnnouncedByMe.remove(segment);
            announceCount.decrementAndGet();
        }

        @Override
        public void announceSegments(Iterable<DataSegment> segments) {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.add(segment);
            }
            announceCount.addAndGet(Iterables.size(segments));
        }

        @Override
        public void unannounceSegments(Iterable<DataSegment> segments) {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.remove(segment);
            }
            announceCount.addAndGet(-Iterables.size(segments));
        }
    };
    segmentLoaderConfig = new SegmentLoaderConfig() {

        @Override
        public File getInfoDir() {
            return testStorageLocation.getInfoDir();
        }

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 50;
        }

        @Override
        public List<StorageLocationConfig> getLocations() {
            return locations;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    };
    noAnnouncerSegmentLoaderConfig = new SegmentLoaderConfig() {

        @Override
        public File getInfoDir() {
            return testStorageLocation.getInfoDir();
        }

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 0;
        }

        @Override
        public List<StorageLocationConfig> getLocations() {
            return locations;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    };
    segmentLoaderConfigNoLocations = new SegmentLoaderConfig() {

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 50;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    };
    scheduledExecutorFactory = new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int corePoolSize, String nameFormat) {
            /*
               Override normal behavoir by adding the runnable to a list so that you can make sure
               all the shceduled runnables are executed by explicitly calling run() on each item in the list
             */
            return new ScheduledThreadPoolExecutor(corePoolSize, Execs.makeThreadFactory(nameFormat)) {

                @Override
                public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
                    scheduledRunnable.add(command);
                    return null;
                }
            };
        }
    };
    segmentLoadDropHandler = new SegmentLoadDropHandler(jsonMapper, segmentLoaderConfig, announcer, Mockito.mock(DataSegmentServerAnnouncer.class), segmentManager, segmentCacheManager, scheduledExecutorFactory.create(5, "SegmentLoadDropHandlerTest-[%d]"), new ServerTypeConfig(ServerType.HISTORICAL));
}