Examples with MiniZooKeeperCluster org.apache.hive.testutils.MiniZooKeeperCluster used on opensource projects

Example 1 with MiniZooKeeperCluster

use of org.apache.hive.testutils.MiniZooKeeperCluster in project hive by apache.

the class MiniLlapCluster method serviceInit.

@Override
public void serviceInit(Configuration conf) throws IOException, InterruptedException {
    int rpcPort = 0;
    int externalClientCloudRpcPort = 0;
    int mngPort = 0;
    int shufflePort = 0;
    int webPort = 0;
    int outputFormatServicePort = 0;
    boolean usePortsFromConf = conf.getBoolean("minillap.usePortsFromConf", false);
    LOG.info("MiniLlap configured to use ports from conf: {}", usePortsFromConf);
    if (usePortsFromConf) {
        rpcPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT);
        externalClientCloudRpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
        mngPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT);
        shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT);
        webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
        outputFormatServicePort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
    }
    HiveConf.setIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT, outputFormatServicePort);
    if (ownZkCluster) {
        miniZooKeeperCluster = new MiniZooKeeperCluster();
        miniZooKeeperCluster.startup(zkWorkDir);
    } else {
    // Already setup in the create method
    }
    conf.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    conf.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    conf.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    // Also add ZK settings to clusterSpecificConf to make sure these get picked up by whoever started this.
    clusterSpecificConfiguration.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(conf);
    LOG.info("Initializing {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
    for (int i = 0; i < numInstances; i++) {
        llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService, execBytesPerService, llapIoEnabled, ioIsDirect, ioBytesPerService, localDirs, rpcPort, externalClientCloudSetupEnabled, externalClientCloudRpcPort, mngPort, shufflePort, webPort, clusterNameTrimmed);
        llapDaemons[i].init(new Configuration(conf));
    }
    LOG.info("Initialized {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
}

Example 2 with MiniZooKeeperCluster

use of org.apache.hive.testutils.MiniZooKeeperCluster in project hive by apache.

the class InformationSchemaWithPrivilegeTestBase method setupInternal.

public static void setupInternal(boolean zookeeperSSLEnabled) throws Exception {
    File zkDataDir = new File(System.getProperty("test.tmp.dir"));
    zkCluster = new MiniZooKeeperCluster(zookeeperSSLEnabled);
    int zkPort = zkCluster.startup(zkDataDir);
    miniHS2 = new MiniHS2(new HiveConf());
    confOverlay = new HashMap<String, String>();
    Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp"));
    confOverlay.put("mapred.local.dir", workDir + File.separator + "TestInformationSchemaWithPrivilege" + File.separator + "mapred" + File.separator + "local");
    confOverlay.put("mapred.system.dir", workDir + File.separator + "TestInformationSchemaWithPrivilege" + File.separator + "mapred" + File.separator + "system");
    confOverlay.put("mapreduce.jobtracker.staging.root.dir", workDir + File.separator + "TestInformationSchemaWithPrivilege" + File.separator + "mapred" + File.separator + "staging");
    confOverlay.put("mapred.temp.dir", workDir + File.separator + "TestInformationSchemaWithPrivilege" + File.separator + "mapred" + File.separator + "temp");
    confOverlay.put(ConfVars.HIVE_PRIVILEGE_SYNCHRONIZER_INTERVAL.varname, "1");
    confOverlay.put(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY.varname, "true");
    confOverlay.put(ConfVars.HIVE_AUTHORIZATION_MANAGER.varname, TestHiveAuthorizerFactory.class.getName());
    confOverlay.put(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    confOverlay.put(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, Integer.toString(zkPort));
    confOverlay.put(MetastoreConf.ConfVars.AUTO_CREATE_ALL.getVarname(), "true");
    confOverlay.put(ConfVars.HIVE_AUTHENTICATOR_MANAGER.varname, FakeGroupAuthenticator.class.getName());
    confOverlay.put(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "true");
    confOverlay.put(ConfVars.HIVE_PRIVILEGE_SYNCHRONIZER.varname, "true");
    confOverlay.put(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST.varname, ".*");
    if (zookeeperSSLEnabled) {
        String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty("test.data.files") : (new HiveConf()).get("test.data.files").replace('\\', '/').replace("c:", "");
        confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION.varname, dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
        confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD);
        confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION.varname, dataFileDir + File.separator + TRUST_STORE_NAME);
        confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD);
        confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE.varname, "true");
    }
    miniHS2.start(confOverlay);
}

Example 3 with MiniZooKeeperCluster

use of org.apache.hive.testutils.MiniZooKeeperCluster in project hive by apache.

the class ZooKeeperTokenStoreTestBase method setUpInternal.

public static void setUpInternal(boolean sslEnabled) throws Exception {
    File zkDataDir = new File(System.getProperty("test.tmp.dir"));
    if (zkCluster != null) {
        throw new IOException("Cluster already running");
    }
    zkCluster = new MiniZooKeeperCluster(sslEnabled);
    zkPort = zkCluster.startup(zkDataDir);
    zkSslEnabled = sslEnabled;
}

Example 4 with MiniZooKeeperCluster

use of org.apache.hive.testutils.MiniZooKeeperCluster in project hive by apache.

the class ManyMiniCluster method setupZookeeper.

private void setupZookeeper() {
    try {
        zookeeperDir = new File(workDir, "zk").getAbsolutePath();
        zookeeperPort = findFreePort();
        zookeeperCluster = new MiniZooKeeperCluster();
        zookeeperCluster.setDefaultClientPort(zookeeperPort);
        zookeeperCluster.startup(new File(zookeeperDir));
    } catch (Exception e) {
        throw new IllegalStateException("Failed to Setup Zookeeper Cluster", e);
    }
}