Examples with Announcer io.druid.curator.announcement.Announcer used on opensource projects

Example 1 with Announcer

use of io.druid.curator.announcement.Announcer in project druid by druid-io.

the class BatchServerInventoryViewTest method setUp.

@Before
public void setUp() throws Exception {
    testingCluster = new TestingCluster(1);
    testingCluster.start();
    cf = CuratorFrameworkFactory.builder().connectString(testingCluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(1, 10)).compressionProvider(new PotentiallyGzippedCompressionProvider(true)).build();
    cf.start();
    cf.blockUntilConnected();
    cf.create().creatingParentsIfNeeded().forPath(testBasePath);
    jsonMapper = new DefaultObjectMapper();
    announcer = new Announcer(cf, MoreExecutors.sameThreadExecutor());
    announcer.start();
    segmentAnnouncer = new BatchDataSegmentAnnouncer(new DruidServerMetadata("id", "host", Long.MAX_VALUE, "type", "tier", 0), new BatchDataSegmentAnnouncerConfig() {

        @Override
        public int getSegmentsPerNode() {
            return 50;
        }
    }, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, announcer, jsonMapper);
    segmentAnnouncer.start();
    testSegments = Sets.newConcurrentHashSet();
    for (int i = 0; i < INITIAL_SEGMENTS; i++) {
        testSegments.add(makeSegment(i));
    }
    batchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, cf, jsonMapper, Predicates.<Pair<DruidServerMetadata, DataSegment>>alwaysTrue());
    batchServerInventoryView.start();
    inventoryUpdateCounter.set(0);
    filteredBatchServerInventoryView = new BatchServerInventoryView(new ZkPathsConfig() {

        @Override
        public String getBase() {
            return testBasePath;
        }
    }, cf, jsonMapper, new Predicate<Pair<DruidServerMetadata, DataSegment>>() {

        @Override
        public boolean apply(@Nullable Pair<DruidServerMetadata, DataSegment> input) {
            return input.rhs.getInterval().getStart().isBefore(SEGMENT_INTERVAL_START.plusDays(INITIAL_SEGMENTS));
        }
    }) {

        @Override
        protected DruidServer addInnerInventory(DruidServer container, String inventoryKey, Set<DataSegment> inventory) {
            DruidServer server = super.addInnerInventory(container, inventoryKey, inventory);
            inventoryUpdateCounter.incrementAndGet();
            return server;
        }
    };
    filteredBatchServerInventoryView.start();
}

Example 2 with Announcer

use of io.druid.curator.announcement.Announcer in project druid by druid-io.

the class ListenerResourceAnnouncerTest method testStartCorrect.

@Test
public void testStartCorrect() throws Exception {
    final Announcer announcer = EasyMock.createStrictMock(Announcer.class);
    final HostAndPort node = HostAndPort.fromString("some_host");
    final ListenerResourceAnnouncer resourceAnnouncer = new ListenerResourceAnnouncer(announcer, listeningAnnouncerConfig, listenerKey, node) {
    };
    announcer.announce(EasyMock.eq(ZKPaths.makePath(announcePath, node.getHostText())), EasyMock.aryEq(resourceAnnouncer.getAnnounceBytes()));
    EasyMock.expectLastCall().once();
    EasyMock.replay(announcer);
    resourceAnnouncer.start();
    EasyMock.verify(announcer);
}

Example 3 with Announcer

use of io.druid.curator.announcement.Announcer in project druid by druid-io.

the class ListenerDiscovererTest method testFullService.

@Test(timeout = 60_000L)
public void testFullService() throws Exception {
    final String listenerKey = "listenerKey";
    final String listenerTier = "listenerTier";
    final String listenerTierChild = "tierChild";
    final String tierZkPath = ZKPaths.makePath(listenerTier, listenerTierChild);
    setupServerAndCurator();
    final ExecutorService executorService = Execs.singleThreaded("listenerDiscovererTest--%s");
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            executorService.shutdownNow();
        }
    });
    closerRule.closeLater(server);
    closerRule.closeLater(curator);
    curator.start();
    curator.blockUntilConnected(10, TimeUnit.SECONDS);
    Assert.assertEquals("/druid", curator.create().forPath("/druid"));
    final Announcer announcer = new Announcer(curator, executorService);
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            announcer.stop();
        }
    });
    final ListeningAnnouncerConfig config = new ListeningAnnouncerConfig(new ZkPathsConfig());
    final ListenerDiscoverer listenerDiscoverer = new ListenerDiscoverer(curator, config);
    listenerDiscoverer.start();
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            listenerDiscoverer.stop();
        }
    });
    Assert.assertTrue(listenerDiscoverer.getNodes(listenerKey).isEmpty());
    final HostAndPort node = HostAndPort.fromParts("someHost", 8888);
    final ListenerResourceAnnouncer listenerResourceAnnouncer = new ListenerResourceAnnouncer(announcer, config, listenerKey, node) {
    };
    listenerResourceAnnouncer.start();
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            listenerResourceAnnouncer.stop();
        }
    });
    final ListenerResourceAnnouncer tieredListenerResourceAnnouncer = new ListenerResourceAnnouncer(announcer, config, tierZkPath, node) {
    };
    tieredListenerResourceAnnouncer.start();
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            tieredListenerResourceAnnouncer.stop();
        }
    });
    announcer.start();
    Assert.assertNotNull(curator.checkExists().forPath(config.getAnnouncementPath(listenerKey)));
    // Have to wait for background syncing
    while (listenerDiscoverer.getNodes(listenerKey).isEmpty()) {
        // Will timeout at test's timeout setting
        Thread.sleep(1);
    }
    Assert.assertEquals(ImmutableSet.of(HostAndPort.fromString(node.toString())), listenerDiscoverer.getNodes(listenerKey));
    // 2nd call of two concurrent getNewNodes should return no entry collection
    listenerDiscoverer.getNewNodes(listenerKey);
    Assert.assertEquals(0, listenerDiscoverer.getNewNodes(listenerKey).size());
    Assert.assertEquals(ImmutableSet.of(listenerKey, listenerTier), ImmutableSet.copyOf(listenerDiscoverer.discoverChildren(null)));
    Assert.assertEquals(ImmutableSet.of(listenerTierChild), ImmutableSet.copyOf(listenerDiscoverer.discoverChildren(listenerTier)));
}

Example 4 with Announcer

use of io.druid.curator.announcement.Announcer in project druid by druid-io.

the class ListenerResourceAnnouncerTest method testAnnouncerBehaves.

@Test
public void testAnnouncerBehaves() throws Exception {
    setupServerAndCurator();
    closerRule.closeLater(server);
    curator.start();
    closerRule.closeLater(curator);
    Assert.assertNotNull(curator.create().forPath("/druid"));
    Assert.assertTrue(curator.blockUntilConnected(10, TimeUnit.SECONDS));
    final Announcer announcer = new Announcer(curator, executorService);
    final HostAndPort node = HostAndPort.fromString("localhost");
    final ListenerResourceAnnouncer listenerResourceAnnouncer = new ListenerResourceAnnouncer(announcer, listeningAnnouncerConfig, listenerKey, node) {
    };
    listenerResourceAnnouncer.start();
    announcer.start();
    closerRule.closeLater(new Closeable() {

        @Override
        public void close() throws IOException {
            announcer.stop();
        }
    });
    Assert.assertNotNull(curator.checkExists().forPath(announcePath));
    final String nodePath = ZKPaths.makePath(announcePath, node.getHostText());
    Assert.assertNotNull(curator.checkExists().forPath(nodePath));
    Assert.assertEquals(Longs.BYTES, curator.getData().decompressed().forPath(nodePath).length);
    Assert.assertNull(curator.checkExists().forPath(listeningAnnouncerConfig.getAnnouncementPath(listenerKey + "FOO")));
    listenerResourceAnnouncer.stop();
    listenerResourceAnnouncer.start();
    listenerResourceAnnouncer.start();
    listenerResourceAnnouncer.stop();
    listenerResourceAnnouncer.stop();
    listenerResourceAnnouncer.start();
    listenerResourceAnnouncer.stop();
    listenerResourceAnnouncer.start();
    listenerResourceAnnouncer.stop();
    Assert.assertNull(curator.checkExists().forPath(nodePath));
}

Example 5 with Announcer

use of io.druid.curator.announcement.Announcer in project druid by druid-io.

the class ZkCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    try {
        infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest");
        infoDir.mkdirs();
        for (File file : infoDir.listFiles()) {
            file.delete();
        }
        log.info("Creating tmp test files in [%s]", infoDir);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    scheduledRunnable = Lists.newArrayList();
    segmentLoader = new CacheTestSegmentLoader();
    serverManager = new ServerManager(segmentLoader, new NoopQueryRunnerFactoryConglomerate(), new NoopServiceEmitter(), MoreExecutors.sameThreadExecutor(), MoreExecutors.sameThreadExecutor(), new DefaultObjectMapper(), new LocalCacheProvider().get(), new CacheConfig());
    final ZkPathsConfig zkPaths = new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "/druid";
        }
    };
    segmentsAnnouncedByMe = new ConcurrentSkipListSet<>();
    announceCount = new AtomicInteger(0);
    announcer = new DataSegmentAnnouncer() {

        private final DataSegmentAnnouncer delegate = new BatchDataSegmentAnnouncer(me, new BatchDataSegmentAnnouncerConfig(), zkPaths, new Announcer(curator, Execs.singleThreaded("blah")), jsonMapper);

        @Override
        public void announceSegment(DataSegment segment) throws IOException {
            segmentsAnnouncedByMe.add(segment);
            announceCount.incrementAndGet();
            delegate.announceSegment(segment);
        }

        @Override
        public void unannounceSegment(DataSegment segment) throws IOException {
            segmentsAnnouncedByMe.remove(segment);
            announceCount.decrementAndGet();
            delegate.unannounceSegment(segment);
        }

        @Override
        public void announceSegments(Iterable<DataSegment> segments) throws IOException {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.add(segment);
            }
            announceCount.addAndGet(Iterables.size(segments));
            delegate.announceSegments(segments);
        }

        @Override
        public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.remove(segment);
            }
            announceCount.addAndGet(-Iterables.size(segments));
            delegate.unannounceSegments(segments);
        }

        @Override
        public boolean isAnnounced(DataSegment segment) {
            return segmentsAnnouncedByMe.contains(segment);
        }
    };
    zkCoordinator = new ZkCoordinator(jsonMapper, new SegmentLoaderConfig() {

        @Override
        public File getInfoDir() {
            return infoDir;
        }

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 50;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    }, zkPaths, me, announcer, curator, serverManager, new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int corePoolSize, String nameFormat) {
            /*
               Override normal behavoir by adding the runnable to a list so that you can make sure
               all the shceduled runnables are executed by explicitly calling run() on each item in the list
             */
            return new ScheduledThreadPoolExecutor(corePoolSize, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(nameFormat).build()) {

                @Override
                public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
                    scheduledRunnable.add(command);
                    return null;
                }
            };
        }
    });
}