Examples with MiniLlapCluster org.apache.hadoop.hive.llap.daemon.MiniLlapCluster used on opensource projects

Example 1 with MiniLlapCluster

use of org.apache.hadoop.hive.llap.daemon.MiniLlapCluster in project hive by apache.

the class LlapItUtils method startAndGetMiniLlapCluster.

public static MiniLlapCluster startAndGetMiniLlapCluster(Configuration conf, MiniZooKeeperCluster miniZkCluster, String confDir) throws IOException {
    MiniLlapCluster llapCluster;
    LOG.info("Using conf dir: {}", confDir);
    if (confDir != null && !confDir.isEmpty()) {
        conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml"));
    }
    Configuration daemonConf = new LlapDaemonConfiguration(conf);
    final String clusterName = "llap";
    final long maxMemory = LlapDaemon.getTotalHeapSize();
    // 15% for io cache
    final long memoryForCache = (long) (0.15f * maxMemory);
    // 75% for 4 executors
    final long totalExecutorMemory = (long) (0.75f * maxMemory);
    final int numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
    final boolean asyncIOEnabled = true;
    // enabling this will cause test failures in Mac OS X
    final boolean directMemoryEnabled = false;
    final int numLocalDirs = 1;
    LOG.info("MiniLlap Configs -  maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled + " numLocalDirs: " + numLocalDirs);
    llapCluster = MiniLlapCluster.create(clusterName, miniZkCluster, 1, numExecutors, totalExecutorMemory, asyncIOEnabled, directMemoryEnabled, memoryForCache, numLocalDirs);
    llapCluster.init(daemonConf);
    llapCluster.start();
    // Augment conf with the settings from the started llap configuration.
    Configuration llapConf = llapCluster.getClusterSpecificConfiguration();
    Iterator<Map.Entry<String, String>> confIter = llapConf.iterator();
    while (confIter.hasNext()) {
        Map.Entry<String, String> entry = confIter.next();
        conf.set(entry.getKey(), entry.getValue());
    }
    return llapCluster;
}