Examples with SessionState org.apache.hadoop.hive.ql.session.SessionState used on opensource projects

Example 91 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class LineageLogger method run.

@Override
public void run(HookContext hookContext) {
    assert (hookContext.getHookType() == HookType.POST_EXEC_HOOK);
    QueryPlan plan = hookContext.getQueryPlan();
    Index index = hookContext.getIndex();
    SessionState ss = SessionState.get();
    if (ss != null && index != null && OPERATION_NAMES.contains(plan.getOperationName()) && !plan.isExplain()) {
        try {
            StringBuilderWriter out = new StringBuilderWriter(1024);
            JsonWriter writer = new JsonWriter(out);
            String queryStr = plan.getQueryStr().trim();
            writer.beginObject();
            writer.name("version").value(FORMAT_VERSION);
            HiveConf conf = ss.getConf();
            boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST);
            if (!testMode) {
                // Don't emit user/timestamp info in test mode,
                // so that the test golden output file is fixed.
                long queryTime = plan.getQueryStartTime().longValue();
                if (queryTime == 0)
                    queryTime = System.currentTimeMillis();
                long duration = System.currentTimeMillis() - queryTime;
                writer.name("user").value(hookContext.getUgi().getUserName());
                writer.name("timestamp").value(queryTime / 1000);
                writer.name("duration").value(duration);
                writer.name("jobIds");
                writer.beginArray();
                List<TaskRunner> tasks = hookContext.getCompleteTaskList();
                if (tasks != null && !tasks.isEmpty()) {
                    for (TaskRunner task : tasks) {
                        String jobId = task.getTask().getJobID();
                        if (jobId != null) {
                            writer.value(jobId);
                        }
                    }
                }
                writer.endArray();
            }
            writer.name("engine").value(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
            writer.name("database").value(ss.getCurrentDatabase());
            writer.name("hash").value(getQueryHash(queryStr));
            writer.name("queryText").value(queryStr);
            List<Edge> edges = getEdges(plan, index);
            Set<Vertex> vertices = getVertices(edges);
            writeEdges(writer, edges, hookContext.getConf());
            writeVertices(writer, vertices);
            writer.endObject();
            writer.close();
            // Logger the lineage info
            String lineage = out.toString();
            if (testMode) {
                // Logger to console
                log(lineage);
            } else {
                // In non-test mode, emit to a log file,
                // which can be different from the normal hive.log.
                // For example, using NoDeleteRollingFileAppender to
                // log to some file with different rolling policy.
                LOG.info(lineage);
            }
        } catch (Throwable t) {
            // Don't fail the query just because of any lineage issue.
            log("Failed to log lineage graph, query is not affected\n" + org.apache.hadoop.util.StringUtils.stringifyException(t));
        }
    }
}

Example 92 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class PostExecOrcRowGroupCountPrinter method run.

@Override
public void run(HookContext hookContext) throws Exception {
    assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK);
    HiveConf conf = hookContext.getConf();
    if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
        return;
    }
    LOG.info("Executing post execution hook to print ORC row groups read counter..");
    SessionState ss = SessionState.get();
    SessionState.LogHelper console = ss.getConsole();
    QueryPlan plan = hookContext.getQueryPlan();
    if (plan == null) {
        return;
    }
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        LOG.info("Printing ORC row group counter for tez task: " + tezTask.getName());
        TezCounters counters = tezTask.getTezCounters();
        if (counters != null) {
            for (CounterGroup group : counters) {
                if (group.getName().equals(LlapIOCounters.class.getName())) {
                    console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false);
                    for (TezCounter counter : group) {
                        if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) {
                            console.printInfo("   " + counter.getDisplayName() + ": " + counter.getValue(), false);
                        }
                    }
                }
            }
        }
    }
}

Example 93 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class HCatCli method main.

@SuppressWarnings("static-access")
public static void main(String[] args) {
    try {
        LogUtils.initHiveLog4j();
    } catch (LogInitializationException e) {
    }
    LOG = LoggerFactory.getLogger(HCatCli.class);
    CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
    ss.in = System.in;
    try {
        ss.out = new PrintStream(System.out, true, "UTF-8");
        ss.err = new PrintStream(System.err, true, "UTF-8");
    } catch (UnsupportedEncodingException e) {
        System.exit(1);
    }
    HiveConf conf = ss.getConf();
    HiveConf.setVar(conf, ConfVars.SEMANTIC_ANALYZER_HOOK, HCatSemanticAnalyzer.class.getName());
    String engine = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE);
    final String MR_ENGINE = "mr";
    if (!MR_ENGINE.equalsIgnoreCase(engine)) {
        HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_ENGINE, MR_ENGINE);
        LOG.info("Forcing " + ConfVars.HIVE_EXECUTION_ENGINE + " to " + MR_ENGINE);
    }
    Options options = new Options();
    // -e 'quoted-query-string'
    options.addOption(OptionBuilder.hasArg().withArgName("exec").withDescription("hcat command given from command line").create('e'));
    // -f <query-file>
    options.addOption(OptionBuilder.hasArg().withArgName("file").withDescription("hcat commands in file").create('f'));
    // -g
    options.addOption(OptionBuilder.hasArg().withArgName("group").withDescription("group for the db/table specified in CREATE statement").create('g'));
    // -p
    options.addOption(OptionBuilder.hasArg().withArgName("perms").withDescription("permissions for the db/table specified in CREATE statement").create('p'));
    // -D
    options.addOption(OptionBuilder.hasArgs(2).withArgName("property=value").withValueSeparator().withDescription("use hadoop value for given property").create('D'));
    // [-h|--help]
    options.addOption(new Option("h", "help", false, "Print help information"));
    Parser parser = new GnuParser();
    CommandLine cmdLine = null;
    try {
        cmdLine = parser.parse(options, args);
    } catch (ParseException e) {
        printUsage(options, System.err);
        // Note, we print to System.err instead of ss.err, because if we can't parse our
        // commandline, we haven't even begun, and therefore cannot be expected to have
        // reasonably constructed or started the SessionState.
        System.exit(1);
    }
    // -D : process these first, so that we can instantiate SessionState appropriately.
    setConfProperties(conf, cmdLine.getOptionProperties("D"));
    // -h
    if (cmdLine.hasOption('h')) {
        printUsage(options, ss.out);
        sysExit(ss, 0);
    }
    // -e
    String execString = (String) cmdLine.getOptionValue('e');
    // -f
    String fileName = (String) cmdLine.getOptionValue('f');
    if (execString != null && fileName != null) {
        ss.err.println("The '-e' and '-f' options cannot be specified simultaneously");
        printUsage(options, ss.err);
        sysExit(ss, 1);
    }
    // -p
    String perms = (String) cmdLine.getOptionValue('p');
    if (perms != null) {
        validatePermissions(ss, conf, perms);
    }
    // -g
    String grp = (String) cmdLine.getOptionValue('g');
    if (grp != null) {
        conf.set(HCatConstants.HCAT_GROUP, grp);
    }
    // Now that the properties are in, we can instantiate SessionState.
    SessionState.start(ss);
    if (execString != null) {
        sysExit(ss, processLine(execString));
    }
    try {
        if (fileName != null) {
            sysExit(ss, processFile(fileName));
        }
    } catch (FileNotFoundException e) {
        ss.err.println("Input file not found. (" + e.getMessage() + ")");
        sysExit(ss, 1);
    } catch (IOException e) {
        ss.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
        sysExit(ss, 1);
    }
    // -h
    printUsage(options, ss.err);
    sysExit(ss, 1);
}

Example 94 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class TezSessionPool method returnSessionInternal.

private boolean returnSessionInternal(SessionType session, boolean isAsync) {
    // Make sure that if the session is returned to the pool, it doesn't live in the global.
    SessionState sessionState = SessionState.get();
    if (sessionState != null) {
        sessionState.setTezSession(null);
    }
    // The session will be restarted and return to us.
    if (!session.stopUsing())
        return true;
    boolean canPutBack = putSessionBack(session, true);
    if (canPutBack)
        return true;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Closing an unneeded returned session " + session);
    }
    // The caller is responsible for destroying the session.
    if (isAsync)
        return false;
    try {
        session.close(false);
    } catch (Exception ex) {
        LOG.error("Failed to close " + session, ex);
    }
    return true;
}

Example 95 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class TezSessionState method getLlapToken.

private static Token<LlapTokenIdentifier> getLlapToken(String user, final Configuration conf) throws IOException {
    // TODO: parts of this should be moved out of TezSession to reuse the clients, but there's
    // no good place for that right now (HIVE-13698).
    // TODO: De-link from SessionState. A TezSession can be linked to different Hive Sessions via the pool.
    SessionState session = SessionState.get();
    boolean isInHs2 = session != null && session.isHiveServerQuery();
    Token<LlapTokenIdentifier> token = null;
    // For Tez, we don't use appId to distinguish the tokens.
    LlapCoordinator coordinator = null;
    if (isInHs2) {
        // We are in HS2, get the token locally.
        // TODO: coordinator should be passed in; HIVE-13698. Must be initialized for now.
        coordinator = LlapCoordinator.getInstance();
        if (coordinator == null) {
            throw new IOException("LLAP coordinator not initialized; cannot get LLAP tokens");
        }
        // Signing is not required for Tez.
        token = coordinator.getLocalTokenClient(conf, user).createToken(null, null, false);
    } else {
        // We are not in HS2; always create a new client for now.
        token = new LlapTokenClient(conf).getDelegationToken(null);
    }
    if (LOG.isInfoEnabled()) {
        LOG.info("Obtained a LLAP token: " + token);
    }
    return token;
}