Examples with ZKClientService org.apache.twill.zookeeper.ZKClientService used on opensource projects

Example 11 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class HBaseQueueTest method init.

@BeforeClass
public static void init() throws Exception {
    hConf = TEST_HBASE.getConfiguration();
    // Customize test configuration
    cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, TEST_HBASE.getZkConnectionString());
    cConf.set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Integer.toString(Networks.getRandomPort()));
    cConf.set(Constants.Dataset.TABLE_PREFIX, TABLE_PREFIX);
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.setLong(QueueConstants.QUEUE_CONFIG_UPDATE_FREQUENCY, 10000L);
    // Test with fewer splits than default (16).
    // Fewer splits make the forceEvict runs faster, which makes all queue tests run faster
    cConf.setInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS, 4);
    cConf.setLong(TxConstants.Manager.CFG_TX_TIMEOUT, 100000000L);
    cConf.setLong(TxConstants.Manager.CFG_TX_MAX_TIMEOUT, 100000000L);
    injector = Guice.createInjector(new DataFabricModules().getDistributedModules(), new ConfigModule(cConf, hConf), new ZKClientModule(), new LocationRuntimeModule().getDistributedModules(), new NamespaceClientUnitTestModule().getModule(), new DiscoveryRuntimeModule().getDistributedModules(), new TransactionMetricsModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class).in(Scopes.SINGLETON);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
        }
    });
    // create HBase namespace
    hbaseAdmin = TEST_HBASE.getHBaseAdmin();
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, hbaseAdmin.getConfiguration()).get();
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NAMESPACE_ID));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NAMESPACE_ID1));
    new ConfigurationWriter(hConf, cConf).write(ConfigurationReader.Type.DEFAULT, cConf);
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    txService = injector.getInstance(TransactionService.class);
    Thread t = new Thread() {

        @Override
        public void run() {
            txService.start();
        }
    };
    t.start();
    // The TransactionManager should be started by the txService.
    // We just want a reference to that so that we can ask for tx snapshot
    txSystemClient = injector.getInstance(TransactionSystemClient.class);
    queueClientFactory = injector.getInstance(QueueClientFactory.class);
    queueAdmin = injector.getInstance(QueueAdmin.class);
    executorFactory = injector.getInstance(TransactionExecutorFactory.class);
}

Example 12 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class HBaseFileStreamAdminTest method init.

@BeforeClass
public static void init() throws Exception {
    InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = testHBase.getConfiguration();
    addCConfProperties(cConf);
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    txManager = TxInMemory.getTransactionManager(injector.getInstance(TransactionSystemClient.class));
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
    authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    txManager.startAndWait();
    streamCoordinatorClient.startAndWait();
}

Example 13 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class KafkaServerMain method init.

@Override
public void init(String[] args) {
    CConfiguration cConf = CConfiguration.create();
    String zkConnectStr = cConf.get(Constants.Zookeeper.QUORUM);
    String zkNamespace = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG);
    if (zkNamespace != null) {
        ZKClientService client = ZKClientService.Builder.of(zkConnectStr).build();
        try {
            Services.startAndWait(client, cConf.getLong(Constants.Zookeeper.CLIENT_STARTUP_TIMEOUT_MILLIS), TimeUnit.MILLISECONDS, String.format("Connection timed out while trying to start ZooKeeper client. Please " + "verify that the ZooKeeper quorum settings are correct in " + "cdap-site.xml. Currently configured as: %s", cConf.get(Constants.Zookeeper.QUORUM)));
            String path = "/" + zkNamespace;
            LOG.info(String.format("Creating zookeeper namespace %s", path));
            ZKOperations.ignoreError(client.create(path, null, CreateMode.PERSISTENT), KeeperException.NodeExistsException.class, path).get();
            client.stopAndWait();
            zkConnectStr = String.format("%s/%s", zkConnectStr, zkNamespace);
        } catch (Exception e) {
            throw Throwables.propagate(e);
        } finally {
            client.stopAndWait();
        }
    }
    kafkaProperties = generateKafkaConfig(cConf);
    int partitions = Integer.parseInt(kafkaProperties.getProperty("num.partitions"), 10);
    Preconditions.checkState(partitions > 0, "Num partitions should be greater than zero.");
    int port = Integer.parseInt(kafkaProperties.getProperty("port"), 10);
    Preconditions.checkState(port > 0, "Port number is invalid.");
    String hostname = kafkaProperties.getProperty("host.name");
    InetAddress address = Networks.resolve(hostname, new InetSocketAddress("localhost", 0).getAddress());
    if (hostname != null) {
        if (address.isAnyLocalAddress()) {
            kafkaProperties.remove("host.name");
            try {
                address = InetAddress.getLocalHost();
            } catch (UnknownHostException e) {
                throw Throwables.propagate(e);
            }
        } else {
            hostname = address.getCanonicalHostName();
            kafkaProperties.setProperty("host.name", hostname);
        }
    }
    if (kafkaProperties.getProperty("broker.id") == null) {
        int brokerId = generateBrokerId(address);
        LOG.info(String.format("Initializing server with broker id %d", brokerId));
        kafkaProperties.setProperty("broker.id", Integer.toString(brokerId));
    }
    if (kafkaProperties.getProperty("zookeeper.connect") == null) {
        kafkaProperties.setProperty("zookeeper.connect", zkConnectStr);
    }
}

Example 14 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class ContextManager method createContext.

// this method is called by the mappers/reducers of jobs launched by Hive.
private static Context createContext(Configuration conf) throws IOException {
    // Create context needs to happen only when running in as a MapReduce job.
    // In other cases, ContextManager will be initialized using saveContext method.
    CConfiguration cConf = ConfigurationUtil.get(conf, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE);
    Configuration hConf = ConfigurationUtil.get(conf, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE);
    Injector injector = createInjector(cConf, hConf);
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    StreamAdmin streamAdmin = injector.getInstance(StreamAdmin.class);
    SystemDatasetInstantiatorFactory datasetInstantiatorFactory = injector.getInstance(SystemDatasetInstantiatorFactory.class);
    AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
    AuthorizationEnforcer authorizationEnforcer = injector.getInstance(AuthorizationEnforcer.class);
    return new Context(datasetFramework, streamAdmin, zkClientService, datasetInstantiatorFactory, authenticationContext, authorizationEnforcer);
}

Example 15 with ZKClientService

use of org.apache.twill.zookeeper.ZKClientService in project cdap by caskdata.

the class ZKRouteStoreTest method init.

@BeforeClass
public static void init() throws IOException {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule());
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
}