Examples with ZKClientService org.apache.twill.zookeeper.ZKClientService used on opensource projects

Example 21 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class HBaseConsumerStateTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    }, new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    stateStoreFactory = injector.getInstance(StreamConsumerStateStoreFactory.class);
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    txService = TxInMemory.getTransactionManager(injector.getInstance(TransactionSystemClient.class));
    txService.startAndWait();
}

Example 22 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class HBaseStreamConsumerTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    consumerFactory = injector.getInstance(StreamConsumerFactory.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    txManager = TxInMemory.getTransactionManager(txClient);
    queueClientFactory = injector.getInstance(QueueClientFactory.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    txManager.startAndWait();
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}

Example 23 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithSharedZKClient.

@Test
public void testWithSharedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start both kafka and broker services, it shouldn't affect the state of the shared zk client
    kafkaClientService.startAndWait();
    brokerService.startAndWait();
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // It shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Stop both, still shouldn't affect the state of the shared zk client
    kafkaClientService.stopAndWait();
    brokerService.stopAndWait();
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    // It still shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    zkClientService.stopAndWait();
}

Example 24 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithDedicatedZKClient.

@Test
public void testWithDedicatedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    // Set the zk quorum for the kafka client, expects it to create and start/stop it's own zk client service
    cConf.set(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM, kafkaZKConnect);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start the kafka client, it should increase the zk connections by 1
    kafkaClientService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Start the broker service,
    // it shouldn't affect the zk connections, as it share the same zk client with kafka client
    brokerService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // Stop the broker service, it shouldn't affect the zk connections, as it is still used by the kafka client
    brokerService.stopAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Stop the kafka client, the zk connections should be reduced by 1
    kafkaClientService.stopAndWait();
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    zkClientService.stopAndWait();
}

Example 25 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class ZKExtOperationsTest method testGetAndSet.

@Test
public void testGetAndSet() throws Exception {
    String path = "/testGetAndSet";
    ZKClientService zkClient1 = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    ZKClientService zkClient2 = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    zkClient1.startAndWait();
    zkClient2.startAndWait();
    // First a node would get created since no node there.
    ZKExtOperations.updateOrCreate(zkClient1, path, new Function<Integer, Integer>() {

        @Nullable
        @Override
        public Integer apply(@Nullable Integer input) {
            Assert.assertNull(input);
            return 0;
        }
    }, INT_CODEC).get(10, TimeUnit.SECONDS);
    // Use a 2nd client to do modification
    ZKExtOperations.updateOrCreate(zkClient2, path, new Function<Integer, Integer>() {

        @Nullable
        @Override
        public Integer apply(@Nullable Integer input) {
            Assert.assertEquals(0, input.intValue());
            return 1;
        }
    }, INT_CODEC).get(10, TimeUnit.SECONDS);
    // Use both client to do concurrent modification. Make sure they fetched the same value before performing set
    final CyclicBarrier barrier = new CyclicBarrier(2);
    Function<Integer, Integer> modifier = new Function<Integer, Integer>() {

        @Nullable
        @Override
        public Integer apply(@Nullable Integer input) {
            try {
                if (input == 1) {
                    barrier.await();
                    return 2;
                } else if (input == 2) {
                    return 3;
                }
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
            throw new IllegalStateException("Illegal input " + input);
        }
    };
    Future<Integer> future1 = ZKExtOperations.updateOrCreate(zkClient1, path, modifier, INT_CODEC);
    Future<Integer> future2 = ZKExtOperations.updateOrCreate(zkClient2, path, modifier, INT_CODEC);
    int r1 = future1.get(10, TimeUnit.SECONDS);
    int r2 = future2.get(10, TimeUnit.SECONDS);
    // One of the result should be 2, the other should be 3. The order may vary
    Assert.assertTrue((r1 == 2 && r2 == 3) || (r1 == 3 && r2 == 2));
    // Not doing update by returning null in modifier
    Integer result = ZKExtOperations.updateOrCreate(zkClient1, path, new Function<Integer, Integer>() {

        @Nullable
        @Override
        public Integer apply(@Nullable Integer input) {
            return (input == 3) ? null : 4;
        }
    }, INT_CODEC).get();
    Assert.assertNull(result);
    zkClient1.stopAndWait();
    zkClient2.stopAndWait();
}