Examples with ExecType org.apache.pig.ExecType used on opensource projects

Example 1 with ExecType

use of org.apache.pig.ExecType in project shifu by ShifuML.

the class PigExecutor method createPigServer.

private PigServer createPigServer(SourceType sourceType) throws IOException {
    PigServer pigServer = null;
    if (SourceType.HDFS.equals(sourceType)) {
        if (Environment.getProperty("shifu.pig.exectype", "MAPREDUCE").toLowerCase().equals("tez")) {
            if (isTezRunnable()) {
                try {
                    Class<?> tezClazz = Class.forName("org.apache.pig.backend.hadoop.executionengine.tez.TezExecType");
                    log.info("Pig ExecType: TEZ");
                    pigServer = new ShifuPigServer((ExecType) tezClazz.newInstance());
                } catch (Throwable t) {
                    log.info("Pig ExecType: MAPREDUCE");
                    pigServer = new ShifuPigServer(ExecType.MAPREDUCE);
                }
            } else {
                // fall back to mapreduce
                log.info("Pig ExecType: MAPREDUCE");
                pigServer = new ShifuPigServer(ExecType.MAPREDUCE);
            }
        } else {
            log.info("Pig ExecType: MAPREDUCE");
            pigServer = new ShifuPigServer(ExecType.MAPREDUCE);
        }
        String hdpVersion = HDPUtils.getHdpVersionForHDP224();
        if (StringUtils.isNotBlank(hdpVersion)) {
            // for hdp 2.2.4, hdp.version should be set and configuration files should be added to container class
            pigServer.getPigContext().getProperties().put("hdp.version", hdpVersion);
            pigServer.getPigContext().addJar(HDPUtils.findContainingFile("hdfs-site.xml"));
            pigServer.getPigContext().addJar(HDPUtils.findContainingFile("core-site.xml"));
            pigServer.getPigContext().addJar(HDPUtils.findContainingFile("mapred-site.xml"));
            pigServer.getPigContext().addJar(HDPUtils.findContainingFile("yarn-site.xml"));
        }
    } else {
        log.info("ExecType: LOCAL");
        pigServer = new ShifuPigServer(ExecType.LOCAL);
    }
    return pigServer;
}