Search in sources :

Example 1 with BatchServerInventoryView

use of org.apache.druid.client.BatchServerInventoryView in project druid by druid-io.

the class CuratorDruidCoordinatorTest method setupView.

private void setupView() throws Exception {
    baseView = new BatchServerInventoryView(zkPathsConfig, curator, jsonMapper, Predicates.alwaysTrue(), "test") {

        @Override
        public void registerSegmentCallback(Executor exec, final SegmentCallback callback) {
            super.registerSegmentCallback(exec, new SegmentCallback() {

                @Override
                public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) {
                    CallbackAction res = callback.segmentAdded(server, segment);
                    segmentAddedLatch.countDown();
                    return res;
                }

                @Override
                public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) {
                    CallbackAction res = callback.segmentRemoved(server, segment);
                    segmentRemovedLatch.countDown();
                    return res;
                }

                @Override
                public CallbackAction segmentViewInitialized() {
                    CallbackAction res = callback.segmentViewInitialized();
                    segmentViewInitLatch.countDown();
                    return res;
                }
            });
        }
    };
    serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig());
    baseView.start();
    sourceLoadQueuePeon.start();
    destinationLoadQueuePeon.start();
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, baseView, metadataRuleManager, () -> curator, new NoopServiceEmitter(), scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}
Also used : BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) DataSegment(org.apache.druid.timeline.DataSegment) CoordinatorSegmentWatcherConfig(org.apache.druid.client.CoordinatorSegmentWatcherConfig) Executor(java.util.concurrent.Executor) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) DruidNode(org.apache.druid.server.DruidNode) CoordinatorServerView(org.apache.druid.client.CoordinatorServerView) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups)

Example 2 with BatchServerInventoryView

use of org.apache.druid.client.BatchServerInventoryView in project druid by druid-io.

the class BatchServerInventoryViewTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(true)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(TEST_BASE_PATH);
    jsonMapper = TestHelper.makeJsonMapper();
    announcer = new Announcer(cf, Execs.directExecutor());
    announcer.start();
    DruidServerMetadata serverMetadata = new DruidServerMetadata("id", "host", null, Long.MAX_VALUE, ServerType.HISTORICAL, "tier", 0);
    ZkPathsConfig zkPathsConfig = new ZkPathsConfig() {

        @Override
        public String getBase() {
            return TEST_BASE_PATH;
        }
    };
    serverAnnouncer = new CuratorDataSegmentServerAnnouncer(serverMetadata, zkPathsConfig, announcer, jsonMapper);
    serverAnnouncer.announce();
    segmentAnnouncer = new BatchDataSegmentAnnouncer(serverMetadata, new BatchDataSegmentAnnouncerConfig() {

        @Override
        public int getSegmentsPerNode() {
            return 50;
        }
    }, zkPathsConfig, announcer, jsonMapper);
    testSegments = Sets.newConcurrentHashSet();
    for (int i = 0; i < INITIAL_SEGMENTS; i++) {
        testSegments.add(makeSegment(i));
    }
    batchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return TEST_BASE_PATH;
        }
    }, cf, jsonMapper, Predicates.alwaysTrue(), "test");
    batchServerInventoryView.start();
    inventoryUpdateCounter.set(0);
    filteredBatchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return TEST_BASE_PATH;
        }
    }, cf, jsonMapper, new Predicate<Pair<DruidServerMetadata, DataSegment>>() {

        @Override
        public boolean apply(@Nullable Pair<DruidServerMetadata, DataSegment> input) {
            return input.rhs.getInterval().getStart().isBefore(SEGMENT_INTERVAL_START.plusDays(INITIAL_SEGMENTS));
        }
    }, "test") {

        @Override
        protected DruidServer addInnerInventory(DruidServer container, String inventoryKey, Set<DataSegment> inventory) {
            DruidServer server = super.addInnerInventory(container, inventoryKey, inventory);
            inventoryUpdateCounter.incrementAndGet();
            return server;
        }
    };
    filteredBatchServerInventoryView.start();
}
Also used : BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) BatchDataSegmentAnnouncerConfig(org.apache.druid.server.initialization.BatchDataSegmentAnnouncerConfig) DruidServer(org.apache.druid.client.DruidServer) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) PotentiallyGzippedCompressionProvider(org.apache.druid.curator.PotentiallyGzippedCompressionProvider) DataSegment(org.apache.druid.timeline.DataSegment) CuratorDataSegmentServerAnnouncer(org.apache.druid.server.coordination.CuratorDataSegmentServerAnnouncer) Predicate(com.google.common.base.Predicate) TestingCluster(org.apache.curator.test.TestingCluster) Announcer(org.apache.druid.curator.announcement.Announcer) CuratorDataSegmentServerAnnouncer(org.apache.druid.server.coordination.CuratorDataSegmentServerAnnouncer) DataSegmentServerAnnouncer(org.apache.druid.server.coordination.DataSegmentServerAnnouncer) BatchDataSegmentAnnouncer(org.apache.druid.server.coordination.BatchDataSegmentAnnouncer) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) BatchDataSegmentAnnouncer(org.apache.druid.server.coordination.BatchDataSegmentAnnouncer) Nullable(javax.annotation.Nullable) Pair(org.apache.druid.java.util.common.Pair) Before(org.junit.Before)

Example 3 with BatchServerInventoryView

use of org.apache.druid.client.BatchServerInventoryView in project druid by druid-io.

the class DatasourceOptimizerTest method setupViews.

private void setupViews() throws Exception {
    baseView = new BatchServerInventoryView(zkPathsConfig, curator, jsonMapper, Predicates.alwaysTrue(), "test") {

        @Override
        public void registerSegmentCallback(Executor exec, final SegmentCallback callback) {
            super.registerSegmentCallback(exec, new SegmentCallback() {

                @Override
                public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) {
                    return callback.segmentAdded(server, segment);
                }

                @Override
                public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) {
                    return callback.segmentRemoved(server, segment);
                }

                @Override
                public CallbackAction segmentViewInitialized() {
                    return callback.segmentViewInitialized();
                }
            });
        }
    };
    brokerServerView = new BrokerServerView(EasyMock.createMock(QueryToolChestWarehouse.class), EasyMock.createMock(QueryWatcher.class), getSmileMapper(), EasyMock.createMock(HttpClient.class), baseView, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), new NoopServiceEmitter(), new BrokerSegmentWatcherConfig());
    baseView.start();
}
Also used : BatchServerInventoryView(org.apache.druid.client.BatchServerInventoryView) Executor(java.util.concurrent.Executor) HighestPriorityTierSelectorStrategy(org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy) BrokerServerView(org.apache.druid.client.BrokerServerView) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) DruidServerMetadata(org.apache.druid.server.coordination.DruidServerMetadata) DataSegment(org.apache.druid.timeline.DataSegment) BrokerSegmentWatcherConfig(org.apache.druid.client.BrokerSegmentWatcherConfig) RandomServerSelectorStrategy(org.apache.druid.client.selector.RandomServerSelectorStrategy)

Aggregations

BatchServerInventoryView (org.apache.druid.client.BatchServerInventoryView)3 DruidServerMetadata (org.apache.druid.server.coordination.DruidServerMetadata)3 DataSegment (org.apache.druid.timeline.DataSegment)3 Executor (java.util.concurrent.Executor)2 ZkPathsConfig (org.apache.druid.server.initialization.ZkPathsConfig)2 NoopServiceEmitter (org.apache.druid.server.metrics.NoopServiceEmitter)2 Predicate (com.google.common.base.Predicate)1 Nullable (javax.annotation.Nullable)1 ExponentialBackoffRetry (org.apache.curator.retry.ExponentialBackoffRetry)1 TestingCluster (org.apache.curator.test.TestingCluster)1 BrokerSegmentWatcherConfig (org.apache.druid.client.BrokerSegmentWatcherConfig)1 BrokerServerView (org.apache.druid.client.BrokerServerView)1 CoordinatorSegmentWatcherConfig (org.apache.druid.client.CoordinatorSegmentWatcherConfig)1 CoordinatorServerView (org.apache.druid.client.CoordinatorServerView)1 DruidServer (org.apache.druid.client.DruidServer)1 HighestPriorityTierSelectorStrategy (org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy)1 RandomServerSelectorStrategy (org.apache.druid.client.selector.RandomServerSelectorStrategy)1 PotentiallyGzippedCompressionProvider (org.apache.druid.curator.PotentiallyGzippedCompressionProvider)1 Announcer (org.apache.druid.curator.announcement.Announcer)1 NoopServiceAnnouncer (org.apache.druid.curator.discovery.NoopServiceAnnouncer)1