Examples with LlapDaemon org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon used on opensource projects

Example 1 with LlapDaemon

use of org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon in project hive by apache.

the class LlapDaemonExtension method beforeEach.

@Override
public void beforeEach(ExtensionContext context) throws Exception {
    if (!LOCK.tryLock(1, TimeUnit.MINUTES)) {
        throw new IllegalStateException("Lock acquisition failed cause another test is using the LlapDaemon.");
    }
    final String appName = "testLlapDaemon" + context.getUniqueId();
    HiveConf conf = new HiveConf();
    HiveConf.setVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "llap");
    LlapDaemonInfo.initialize(appName, conf);
    daemon = new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false, -1, new String[1], 0, false, 0, 0, 0, 0, appName);
    daemon.init(conf);
    daemon.start();
}

Example 2 with LlapDaemon

use of org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon in project hive by apache.

the class MiniLlapCluster method serviceInit.

@Override
public void serviceInit(Configuration conf) throws IOException, InterruptedException {
    int rpcPort = 0;
    int externalClientCloudRpcPort = 0;
    int mngPort = 0;
    int shufflePort = 0;
    int webPort = 0;
    int outputFormatServicePort = 0;
    boolean usePortsFromConf = conf.getBoolean("minillap.usePortsFromConf", false);
    LOG.info("MiniLlap configured to use ports from conf: {}", usePortsFromConf);
    if (usePortsFromConf) {
        rpcPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT);
        externalClientCloudRpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
        mngPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT);
        shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT);
        webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
        outputFormatServicePort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
    }
    HiveConf.setIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT, outputFormatServicePort);
    if (ownZkCluster) {
        miniZooKeeperCluster = new MiniZooKeeperCluster();
        miniZooKeeperCluster.startup(zkWorkDir);
    } else {
    // Already setup in the create method
    }
    conf.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    conf.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    conf.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    // Also add ZK settings to clusterSpecificConf to make sure these get picked up by whoever started this.
    clusterSpecificConfiguration.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(conf);
    LOG.info("Initializing {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
    for (int i = 0; i < numInstances; i++) {
        llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService, execBytesPerService, llapIoEnabled, ioIsDirect, ioBytesPerService, localDirs, rpcPort, externalClientCloudSetupEnabled, externalClientCloudRpcPort, mngPort, shufflePort, webPort, clusterNameTrimmed);
        llapDaemons[i].init(new Configuration(conf));
    }
    LOG.info("Initialized {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
}