use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.
the class LoadQueuePeonTest method testFailAssign.
@Test
public void testFailAssign() throws Exception {
final DataSegment segment = dataSegmentWithInterval("2014-10-22T00:00:00Z/P1D");
final CountDownLatch loadRequestSignal = new CountDownLatch(1);
final CountDownLatch segmentLoadedSignal = new CountDownLatch(1);
loadQueuePeon = new LoadQueuePeon(curator, LOAD_QUEUE_PATH, jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly
new TestDruidCoordinatorConfig(null, null, null, new Duration(1), null, null, 10, null, false, false, new Duration("PT1s")));
loadQueuePeon.start();
loadQueueCache.getListenable().addListener(new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
loadRequestSignal.countDown();
}
}
});
loadQueueCache.start();
loadQueuePeon.loadSegment(segment, new LoadPeonCallback() {
@Override
public void execute() {
segmentLoadedSignal.countDown();
}
});
String loadRequestPath = ZKPaths.makePath(LOAD_QUEUE_PATH, segment.getIdentifier());
Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestSignal));
Assert.assertNotNull(curator.checkExists().forPath(loadRequestPath));
Assert.assertEquals(segment, ((SegmentChangeRequestLoad) jsonMapper.readValue(curator.getData().decompressed().forPath(loadRequestPath), DataSegmentChangeRequest.class)).getSegment());
// don't simulate completion of load request here
Assert.assertTrue(timing.forWaiting().awaitLatch(segmentLoadedSignal));
Assert.assertEquals(0, loadQueuePeon.getSegmentsToLoad().size());
Assert.assertEquals(0L, loadQueuePeon.getLoadQueueSize());
}
use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.
the class CliCoordinator method getModules.
@Override
protected List<? extends Module> getModules() {
List<Module> modules = new ArrayList<>();
modules.add(new Module() {
@Override
public void configure(Binder binder) {
binder.bindConstant().annotatedWith(Names.named("serviceName")).to(TieredBrokerConfig.DEFAULT_COORDINATOR_SERVICE_NAME);
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
ConfigProvider.bind(binder, DruidCoordinatorConfig.class);
binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class);
JsonConfigProvider.bind(binder, "druid.manager.segments", MetadataSegmentManagerConfig.class);
JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class);
JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class);
JsonConfigProvider.bind(binder, "druid.coordinator.balancer", BalancerStrategyFactory.class);
binder.bind(RedirectFilter.class).in(LazySingleton.class);
if (beOverlord) {
binder.bind(RedirectInfo.class).to(CoordinatorOverlordRedirectInfo.class).in(LazySingleton.class);
} else {
binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class);
}
binder.bind(MetadataSegmentManager.class).toProvider(MetadataSegmentManagerProvider.class).in(ManageLifecycle.class);
binder.bind(MetadataRuleManager.class).toProvider(MetadataRuleManagerProvider.class).in(ManageLifecycle.class);
binder.bind(AuditManager.class).toProvider(AuditManagerProvider.class).in(ManageLifecycle.class);
binder.bind(IndexingServiceClient.class).in(LazySingleton.class);
binder.bind(CoordinatorServerView.class).in(LazySingleton.class);
binder.bind(DruidCoordinator.class);
binder.bind(LookupCoordinatorManager.class).in(ManageLifecycle.class);
binder.bind(ListenerDiscoverer.class).in(ManageLifecycle.class);
LifecycleModule.register(binder, ListenerDiscoverer.class);
LifecycleModule.register(binder, MetadataStorage.class);
LifecycleModule.register(binder, DruidCoordinator.class);
LifecycleModule.register(binder, LookupCoordinatorManager.class);
binder.bind(JettyServerInitializer.class).to(CoordinatorJettyServerInitializer.class);
Jerseys.addResource(binder, CoordinatorResource.class);
Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
Jerseys.addResource(binder, TiersResource.class);
Jerseys.addResource(binder, RulesResource.class);
Jerseys.addResource(binder, ServersResource.class);
Jerseys.addResource(binder, DatasourcesResource.class);
Jerseys.addResource(binder, MetadataResource.class);
Jerseys.addResource(binder, IntervalsResource.class);
Jerseys.addResource(binder, LookupCoordinatorResource.class);
LifecycleModule.register(binder, Server.class);
LifecycleModule.register(binder, DatasourcesResource.class);
ConditionalMultibind.create(properties, binder, DruidCoordinatorHelper.class, CoordinatorIndexingServiceHelper.class).addConditionBinding("druid.coordinator.merge.on", Predicates.equalTo("true"), DruidCoordinatorSegmentMerger.class).addConditionBinding("druid.coordinator.conversion.on", Predicates.equalTo("true"), DruidCoordinatorVersionConverter.class).addConditionBinding("druid.coordinator.kill.on", Predicates.equalTo("true"), DruidCoordinatorSegmentKiller.class);
}
@Provides
@LazySingleton
public LoadQueueTaskMaster getLoadQueueTaskMaster(CuratorFramework curator, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidCoordinatorConfig config) {
return new LoadQueueTaskMaster(curator, jsonMapper, factory.create(1, "Master-PeonExec--%d"), Executors.newSingleThreadExecutor(), config);
}
});
if (beOverlord) {
modules.addAll(new CliOverlord().getModules(false));
}
return modules;
}
use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.
the class CuratorModule method makeCurator.
@Provides
@LazySingleton
public CuratorFramework makeCurator(CuratorConfig config, EnsembleProvider ensembleProvider, Lifecycle lifecycle) throws IOException {
final CuratorFramework framework = CuratorFrameworkFactory.builder().ensembleProvider(ensembleProvider).sessionTimeoutMs(config.getZkSessionTimeoutMs()).retryPolicy(new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES)).compressionProvider(new PotentiallyGzippedCompressionProvider(config.getEnableCompression())).aclProvider(config.getEnableAcl() ? new SecuredACLProvider() : new DefaultACLProvider()).build();
lifecycle.addHandler(new Lifecycle.Handler() {
@Override
public void start() throws Exception {
log.info("Starting Curator");
framework.start();
}
@Override
public void stop() {
log.info("Stopping Curator");
framework.close();
}
});
return framework;
}
use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project druid by druid-io.
the class CuratorInventoryManagerTest method testSanity.
@Test
public void testSanity() throws Exception {
final MapStrategy strategy = new MapStrategy();
CuratorInventoryManager<Map<String, Integer>, Integer> manager = new CuratorInventoryManager<Map<String, Integer>, Integer>(curator, new StringInventoryManagerConfig("/container", "/inventory"), exec, strategy);
curator.start();
curator.blockUntilConnected();
manager.start();
Assert.assertTrue(Iterables.isEmpty(manager.getInventory()));
CountDownLatch containerLatch = new CountDownLatch(1);
strategy.setNewContainerLatch(containerLatch);
curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath("/container/billy", new byte[] {});
Assert.assertTrue(timing.awaitLatch(containerLatch));
strategy.setNewContainerLatch(null);
final Iterable<Map<String, Integer>> inventory = manager.getInventory();
Assert.assertTrue(Iterables.getOnlyElement(inventory).isEmpty());
CountDownLatch inventoryLatch = new CountDownLatch(2);
strategy.setNewInventoryLatch(inventoryLatch);
curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/1", Ints.toByteArray(100));
curator.create().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/bob", Ints.toByteArray(2287));
Assert.assertTrue(timing.awaitLatch(inventoryLatch));
strategy.setNewInventoryLatch(null);
verifyInventory(manager);
CountDownLatch deleteLatch = new CountDownLatch(1);
strategy.setDeadInventoryLatch(deleteLatch);
curator.delete().forPath("/inventory/billy/1");
Assert.assertTrue(timing.awaitLatch(deleteLatch));
strategy.setDeadInventoryLatch(null);
Assert.assertEquals(1, manager.getInventoryValue("billy").size());
Assert.assertEquals(2287, manager.getInventoryValue("billy").get("bob").intValue());
inventoryLatch = new CountDownLatch(1);
strategy.setNewInventoryLatch(inventoryLatch);
curator.create().withMode(CreateMode.EPHEMERAL).forPath("/inventory/billy/1", Ints.toByteArray(100));
Assert.assertTrue(timing.awaitLatch(inventoryLatch));
strategy.setNewInventoryLatch(null);
verifyInventory(manager);
final CountDownLatch latch = new CountDownLatch(1);
curator.getCuratorListenable().addListener(new CuratorListener() {
@Override
public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception {
if (event.getType() == CuratorEventType.WATCHED && event.getWatchedEvent().getState() == Watcher.Event.KeeperState.Disconnected) {
latch.countDown();
}
}
});
server.stop();
Assert.assertTrue(timing.awaitLatch(latch));
verifyInventory(manager);
// Wait a bit
Thread.sleep(50);
verifyInventory(manager);
}
use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project hadoop by apache.
the class TestChildReaper method testNamespace.
@Test
public void testNamespace() throws Exception {
Timing timing = new Timing();
ChildReaper reaper = null;
CuratorFramework client = CuratorFrameworkFactory.builder().connectString(server.getConnectString()).sessionTimeoutMs(timing.session()).connectionTimeoutMs(timing.connection()).retryPolicy(new RetryOneTime(1)).namespace("foo").build();
try {
client.start();
for (int i = 0; i < 10; ++i) {
client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
}
reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
reaper.start();
timing.forWaiting().sleepABit();
Stat stat = client.checkExists().forPath("/test");
Assert.assertEquals(stat.getNumChildren(), 0);
stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
Assert.assertNotNull(stat);
Assert.assertEquals(stat.getNumChildren(), 0);
} finally {
CloseableUtils.closeQuietly(reaper);
CloseableUtils.closeQuietly(client);
}
}
Aggregations