Examples with HadoopShimsLoader org.apache.tez.hadoop.shim.HadoopShimsLoader used on opensource projects

Example 6 with HadoopShimsLoader

use of org.apache.tez.hadoop.shim.HadoopShimsLoader in project tez by apache.

the class TezExampleBase method run.

/**
 * Utility method to use the example from within code or a test.
 *
 * @param conf      the tez configuration instance which will be used to crate the DAG and
 *                  possible the Tez Client.
 * @param args      arguments to the example
 * @param tezClient an existing running {@link org.apache.tez.client.TezClient} instance if one
 *                  exists. If no TezClient is specified (null), one will be created based on the
 *                  provided configuration. If TezClient is specified, local mode option can not been
 *                  specified in arguments, it takes no effect.
 * @return Zero indicates success, non-zero indicates failure
 * @throws Exception
 */
public int run(TezConfiguration conf, String[] args, @Nullable TezClient tezClient) throws Exception {
    setConf(conf);
    hadoopShim = new HadoopShimsLoader(conf).getHadoopShim();
    GenericOptionsParser optionParser = new GenericOptionsParser(conf, getExtraOptions(), args);
    if (optionParser.getCommandLine().hasOption(LOCAL_MODE)) {
        isLocalMode = true;
        if (tezClient != null) {
            throw new RuntimeException("can't specify local mode when TezClient is created, it takes no effect");
        }
    }
    if (optionParser.getCommandLine().hasOption(DISABLE_SPLIT_GROUPING)) {
        disableSplitGrouping = true;
    }
    if (optionParser.getCommandLine().hasOption(COUNTER_LOG)) {
        isCountersLog = true;
    }
    if (optionParser.getCommandLine().hasOption(GENERATE_SPLIT_IN_CLIENT)) {
        generateSplitInClient = true;
    }
    String[] otherArgs = optionParser.getRemainingArgs();
    return _execute(otherArgs, conf, tezClient);
}

Example 7 with HadoopShimsLoader

use of org.apache.tez.hadoop.shim.HadoopShimsLoader in project tez by apache.

the class TezChild method main.

public static void main(String[] args) throws IOException, InterruptedException, TezException {
    final Configuration defaultConf = new Configuration();
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    final String pid = System.getenv().get("JVM_PID");
    assert args.length == 5;
    String host = args[0];
    int port = Integer.parseInt(args[1]);
    final String containerIdentifier = args[2];
    final String tokenIdentifier = args[3];
    final int attemptNumber = Integer.parseInt(args[4]);
    final String[] localDirs = TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name()));
    LOG.info("TezChild starting with PID=" + pid + ", containerIdentifier=" + containerIdentifier);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Info from cmd line: AM-host: " + host + " AM-port: " + port + " containerIdentifier: " + containerIdentifier + " appAttemptNumber: " + attemptNumber + " tokenIdentifier: " + tokenIdentifier);
    }
    // Security framework already loaded the tokens into current ugi
    DAGProtos.ConfigurationProto confProto = TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
    TezUtilsInternal.addUserSpecifiedTezConfiguration(defaultConf, confProto.getConfKeyValuesList());
    UserGroupInformation.setConfiguration(defaultConf);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    HadoopShim hadoopShim = new HadoopShimsLoader(defaultConf).getHadoopShim();
    // log the system properties
    if (LOG.isInfoEnabled()) {
        String systemPropsToLog = TezCommonUtils.getSystemPropertiesToLog(defaultConf);
        if (systemPropsToLog != null) {
            LOG.info(systemPropsToLog);
        }
    }
    TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier, tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()), System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())), credentials, Runtime.getRuntime().maxMemory(), System.getenv(ApplicationConstants.Environment.USER.toString()), null, true, hadoopShim);
    tezChild.run();
}