Search in sources :

Example 21 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.

the class LoadQueuePeonTest method testFailAssign.

@Test
public void testFailAssign() throws Exception {
    final DataSegment segment = dataSegmentWithInterval("2014-10-22T00:00:00Z/P1D");
    final CountDownLatch loadRequestSignal = new CountDownLatch(1);
    final CountDownLatch segmentLoadedSignal = new CountDownLatch(1);
    loadQueuePeon = new LoadQueuePeon(curator, LOAD_QUEUE_PATH, jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly
    new TestDruidCoordinatorConfig(null, null, null, new Duration(1), null, null, 10, null, false, false, new Duration("PT1s")));
    loadQueuePeon.start();
    loadQueueCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
            if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
                loadRequestSignal.countDown();
            }
        }
    });
    loadQueueCache.start();
    loadQueuePeon.loadSegment(segment, new LoadPeonCallback() {

        @Override
        public void execute() {
            segmentLoadedSignal.countDown();
        }
    });
    String loadRequestPath = ZKPaths.makePath(LOAD_QUEUE_PATH, segment.getIdentifier());
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestSignal));
    Assert.assertNotNull(curator.checkExists().forPath(loadRequestPath));
    Assert.assertEquals(segment, ((SegmentChangeRequestLoad) jsonMapper.readValue(curator.getData().decompressed().forPath(loadRequestPath), DataSegmentChangeRequest.class)).getSegment());
    // don't simulate completion of load request here
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentLoadedSignal));
    Assert.assertEquals(0, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(0L, loadQueuePeon.getLoadQueueSize());
}
Also used : CuratorFramework(org.apache.curator.framework.CuratorFramework) PathChildrenCacheListener(org.apache.curator.framework.recipes.cache.PathChildrenCacheListener) PathChildrenCacheEvent(org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent) Duration(org.joda.time.Duration) CountDownLatch(java.util.concurrent.CountDownLatch) DataSegment(io.druid.timeline.DataSegment) Test(org.junit.Test)

Example 22 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.

the class CliCoordinator method getModules.

@Override
protected List<? extends Module> getModules() {
    List<Module> modules = new ArrayList<>();
    modules.add(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to(TieredBrokerConfig.DEFAULT_COORDINATOR_SERVICE_NAME);
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
            ConfigProvider.bind(binder, DruidCoordinatorConfig.class);
            binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class);
            JsonConfigProvider.bind(binder, "druid.manager.segments", MetadataSegmentManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class);
            JsonConfigProvider.bind(binder, "druid.coordinator.balancer", BalancerStrategyFactory.class);
            binder.bind(RedirectFilter.class).in(LazySingleton.class);
            if (beOverlord) {
                binder.bind(RedirectInfo.class).to(CoordinatorOverlordRedirectInfo.class).in(LazySingleton.class);
            } else {
                binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class);
            }
            binder.bind(MetadataSegmentManager.class).toProvider(MetadataSegmentManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(MetadataRuleManager.class).toProvider(MetadataRuleManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(AuditManager.class).toProvider(AuditManagerProvider.class).in(ManageLifecycle.class);
            binder.bind(IndexingServiceClient.class).in(LazySingleton.class);
            binder.bind(CoordinatorServerView.class).in(LazySingleton.class);
            binder.bind(DruidCoordinator.class);
            binder.bind(LookupCoordinatorManager.class).in(ManageLifecycle.class);
            binder.bind(ListenerDiscoverer.class).in(ManageLifecycle.class);
            LifecycleModule.register(binder, ListenerDiscoverer.class);
            LifecycleModule.register(binder, MetadataStorage.class);
            LifecycleModule.register(binder, DruidCoordinator.class);
            LifecycleModule.register(binder, LookupCoordinatorManager.class);
            binder.bind(JettyServerInitializer.class).to(CoordinatorJettyServerInitializer.class);
            Jerseys.addResource(binder, CoordinatorResource.class);
            Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
            Jerseys.addResource(binder, TiersResource.class);
            Jerseys.addResource(binder, RulesResource.class);
            Jerseys.addResource(binder, ServersResource.class);
            Jerseys.addResource(binder, DatasourcesResource.class);
            Jerseys.addResource(binder, MetadataResource.class);
            Jerseys.addResource(binder, IntervalsResource.class);
            Jerseys.addResource(binder, LookupCoordinatorResource.class);
            LifecycleModule.register(binder, Server.class);
            LifecycleModule.register(binder, DatasourcesResource.class);
            ConditionalMultibind.create(properties, binder, DruidCoordinatorHelper.class, CoordinatorIndexingServiceHelper.class).addConditionBinding("druid.coordinator.merge.on", Predicates.equalTo("true"), DruidCoordinatorSegmentMerger.class).addConditionBinding("druid.coordinator.conversion.on", Predicates.equalTo("true"), DruidCoordinatorVersionConverter.class).addConditionBinding("druid.coordinator.kill.on", Predicates.equalTo("true"), DruidCoordinatorSegmentKiller.class);
        }

        @Provides
        @LazySingleton
        public LoadQueueTaskMaster getLoadQueueTaskMaster(CuratorFramework curator, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidCoordinatorConfig config) {
            return new LoadQueueTaskMaster(curator, jsonMapper, factory.create(1, "Master-PeonExec--%d"), Executors.newSingleThreadExecutor(), config);
        }
    });
    if (beOverlord) {
        modules.addAll(new CliOverlord().getModules(false));
    }
    return modules;
}
Also used : MetadataSegmentManager(io.druid.metadata.MetadataSegmentManager) MetadataRuleManager(io.druid.metadata.MetadataRuleManager) Server(org.eclipse.jetty.server.Server) ArrayList(java.util.ArrayList) CoordinatorResource(io.druid.server.http.CoordinatorResource) LookupCoordinatorResource(io.druid.server.http.LookupCoordinatorResource) ServersResource(io.druid.server.http.ServersResource) ListenerDiscoverer(io.druid.server.listener.announcer.ListenerDiscoverer) Binder(com.google.inject.Binder) LazySingleton(io.druid.guice.LazySingleton) LookupCoordinatorManager(io.druid.server.lookup.cache.LookupCoordinatorManager) TiersResource(io.druid.server.http.TiersResource) CuratorFramework(org.apache.curator.framework.CuratorFramework) DruidCoordinatorConfig(io.druid.server.coordinator.DruidCoordinatorConfig) LookupCoordinatorManagerConfig(io.druid.server.lookup.cache.LookupCoordinatorManagerConfig) AuditManager(io.druid.audit.AuditManager) DruidCoordinator(io.druid.server.coordinator.DruidCoordinator) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) MetadataStorageProvider(io.druid.metadata.MetadataStorageProvider) MetadataResource(io.druid.server.http.MetadataResource) DruidCoordinatorSegmentMerger(io.druid.server.coordinator.helper.DruidCoordinatorSegmentMerger) RulesResource(io.druid.server.http.RulesResource) BalancerStrategyFactory(io.druid.server.coordinator.BalancerStrategyFactory) ManageLifecycle(io.druid.guice.ManageLifecycle) Provides(com.google.inject.Provides) MetadataSegmentManagerConfig(io.druid.metadata.MetadataSegmentManagerConfig) CoordinatorDynamicConfigsResource(io.druid.server.http.CoordinatorDynamicConfigsResource) IntervalsResource(io.druid.server.http.IntervalsResource) DatasourcesResource(io.druid.server.http.DatasourcesResource) MetadataRuleManagerConfig(io.druid.metadata.MetadataRuleManagerConfig) LookupCoordinatorResource(io.druid.server.http.LookupCoordinatorResource) ScheduledExecutorFactory(io.druid.java.util.common.concurrent.ScheduledExecutorFactory) MetadataStorage(io.druid.metadata.MetadataStorage) DruidCoordinatorSegmentKiller(io.druid.server.coordinator.helper.DruidCoordinatorSegmentKiller) RedirectInfo(io.druid.server.http.RedirectInfo) CoordinatorRedirectInfo(io.druid.server.http.CoordinatorRedirectInfo) Module(com.google.inject.Module) LifecycleModule(io.druid.guice.LifecycleModule) LoadQueueTaskMaster(io.druid.server.coordinator.LoadQueueTaskMaster)

Example 23 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.

the class CuratorModule method makeCurator.

@Provides
@LazySingleton
public CuratorFramework makeCurator(CuratorConfig config, EnsembleProvider ensembleProvider, Lifecycle lifecycle) throws IOException {
    final CuratorFramework framework = CuratorFrameworkFactory.builder().ensembleProvider(ensembleProvider).sessionTimeoutMs(config.getZkSessionTimeoutMs()).retryPolicy(new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES)).compressionProvider(new PotentiallyGzippedCompressionProvider(config.getEnableCompression())).aclProvider(config.getEnableAcl() ? new SecuredACLProvider() : new DefaultACLProvider()).build();
    lifecycle.addHandler(new Lifecycle.Handler() {

        @Override
        public void start() throws Exception {
            log.info("Starting Curator");
            framework.start();
        }

        @Override
        public void stop() {
            log.info("Stopping Curator");
            framework.close();
        }
    });
    return framework;
}
Also used : CuratorFramework(org.apache.curator.framework.CuratorFramework) Lifecycle(io.druid.java.util.common.lifecycle.Lifecycle) DefaultACLProvider(org.apache.curator.framework.imps.DefaultACLProvider) IOException(java.io.IOException) BoundedExponentialBackoffRetry(org.apache.curator.retry.BoundedExponentialBackoffRetry) LazySingleton(io.druid.guice.LazySingleton) Provides(com.google.inject.Provides)

Example 24 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.

the class CuratorInventoryManagerTest method testSanity.

@Test
public void testSanity() throws Exception {
    final MapStrategy strategy = new MapStrategy();
    CuratorInventoryManager<Map<String, Integer>, Integer> manager = new CuratorInventoryManager<Map<String, Integer>, Integer>(curator, new StringInventoryManagerConfig("/container", "/inventory"), exec, strategy);
    curator.start();
    curator.blockUntilConnected();
    manager.start();
    Assert.assertTrue(Iterables.isEmpty(manager.getInventory()));
    CountDownLatch containerLatch = new CountDownLatch(1);
    strategy.setNewContainerLatch(containerLatch);
    curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath("/container/billy", new byte[] {});
    Assert.assertTrue(timing.awaitLatch(containerLatch));
    strategy.setNewContainerLatch(null);
    final Iterable<Map<String, Integer>> inventory = manager.getInventory();
    Assert.assertTrue(Iterables.getOnlyElement(inventory).isEmpty());
    CountDownLatch inventoryLatch = new CountDownLatch(2);
    strategy.setNewInventoryLatch(inventoryLatch);
    curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/1", Ints.toByteArray(100));
    curator.create().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/bob", Ints.toByteArray(2287));
    Assert.assertTrue(timing.awaitLatch(inventoryLatch));
    strategy.setNewInventoryLatch(null);
    verifyInventory(manager);
    CountDownLatch deleteLatch = new CountDownLatch(1);
    strategy.setDeadInventoryLatch(deleteLatch);
    curator.delete().forPath("/inventory/billy/1");
    Assert.assertTrue(timing.awaitLatch(deleteLatch));
    strategy.setDeadInventoryLatch(null);
    Assert.assertEquals(1, manager.getInventoryValue("billy").size());
    Assert.assertEquals(2287, manager.getInventoryValue("billy").get("bob").intValue());
    inventoryLatch = new CountDownLatch(1);
    strategy.setNewInventoryLatch(inventoryLatch);
    curator.create().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/1", Ints.toByteArray(100));
    Assert.assertTrue(timing.awaitLatch(inventoryLatch));
    strategy.setNewInventoryLatch(null);
    verifyInventory(manager);
    final CountDownLatch latch = new CountDownLatch(1);
    curator.getCuratorListenable().addListener(new CuratorListener() {

        @Override
        public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception {
            if (event.getType() == CuratorEventType.WATCHED && event.getWatchedEvent().getState() == Watcher.Event.KeeperState.Disconnected) {
                latch.countDown();
            }
        }
    });
    server.stop();
    Assert.assertTrue(timing.awaitLatch(latch));
    verifyInventory(manager);
    // Wait a bit
    Thread.sleep(50);
    verifyInventory(manager);
}
Also used : CuratorEvent(org.apache.curator.framework.api.CuratorEvent) CountDownLatch(java.util.concurrent.CountDownLatch) CuratorFramework(org.apache.curator.framework.CuratorFramework) CuratorListener(org.apache.curator.framework.api.CuratorListener) Map(java.util.Map) Test(org.junit.Test)

Example 25 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project hadoop by apache.

the class TestChildReaper method testNamespace.

@Test
public void testNamespace() throws Exception {
    Timing timing = new Timing();
    ChildReaper reaper = null;
    CuratorFramework client = CuratorFrameworkFactory.builder().connectString(server.getConnectString()).sessionTimeoutMs(timing.session()).connectionTimeoutMs(timing.connection()).retryPolicy(new RetryOneTime(1)).namespace("foo").build();
    try {
        client.start();
        for (int i = 0; i < 10; ++i) {
            client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
        }
        reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
        reaper.start();
        timing.forWaiting().sleepABit();
        Stat stat = client.checkExists().forPath("/test");
        Assert.assertEquals(stat.getNumChildren(), 0);
        stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
        Assert.assertNotNull(stat);
        Assert.assertEquals(stat.getNumChildren(), 0);
    } finally {
        CloseableUtils.closeQuietly(reaper);
        CloseableUtils.closeQuietly(client);
    }
}
Also used : CuratorFramework(org.apache.curator.framework.CuratorFramework) RetryOneTime(org.apache.curator.retry.RetryOneTime) Stat(org.apache.zookeeper.data.Stat) Timing(org.apache.curator.test.Timing) Test(org.junit.Test)

Aggregations

CuratorFramework (org.apache.curator.framework.CuratorFramework)924 Test (org.testng.annotations.Test)290 RetryOneTime (org.apache.curator.retry.RetryOneTime)271 Test (org.junit.Test)199 Timing (org.apache.curator.test.Timing)147 CountDownLatch (java.util.concurrent.CountDownLatch)124 ExponentialBackoffRetry (org.apache.curator.retry.ExponentialBackoffRetry)114 KeeperException (org.apache.zookeeper.KeeperException)93 IOException (java.io.IOException)79 ConnectionState (org.apache.curator.framework.state.ConnectionState)71 CuratorEvent (org.apache.curator.framework.api.CuratorEvent)58 ExecutorService (java.util.concurrent.ExecutorService)55 ConnectionStateListener (org.apache.curator.framework.state.ConnectionStateListener)53 ArrayList (java.util.ArrayList)51 RetryNTimes (org.apache.curator.retry.RetryNTimes)51 RetryPolicy (org.apache.curator.RetryPolicy)41 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)38 Cleanup (lombok.Cleanup)37 BackgroundCallback (org.apache.curator.framework.api.BackgroundCallback)37 Stat (org.apache.zookeeper.data.Stat)36