Examples with HiveStatement org.apache.hive.jdbc.HiveStatement used on opensource projects

Example 1 with HiveStatement

use of org.apache.hive.jdbc.HiveStatement in project zeppelin by apache.

the class HiveUtils method startHiveMonitorThread.

/**
 * Display hive job execution info, and progress info for hive >= 2.3
 *
 * @param stmt
 * @param context
 * @param displayLog
 */
public static void startHiveMonitorThread(Statement stmt, InterpreterContext context, boolean displayLog, JDBCInterpreter jdbcInterpreter) {
    HiveStatement hiveStmt = (HiveStatement) ((DelegatingStatement) ((DelegatingStatement) stmt).getDelegate()).getDelegate();
    String hiveVersion = HiveVersionInfo.getVersion();
    ProgressBar progressBarTemp = null;
    if (isProgressBarSupported(hiveVersion)) {
        LOGGER.debug("ProgressBar is supported for hive version: {}", hiveVersion);
        progressBarTemp = new ProgressBar();
    } else {
        LOGGER.debug("ProgressBar is not supported for hive version: {}", hiveVersion);
    }
    // need to use final variable progressBar in thread, so need progressBarTemp here.
    final ProgressBar progressBar = progressBarTemp;
    final long queryInterval = Long.parseLong(jdbcInterpreter.getProperty("zeppelin.jdbc.hive.monitor.query_interval", DEFAULT_QUERY_PROGRESS_INTERVAL + ""));
    Thread thread = new Thread(() -> {
        String jobUrlTemplate = jdbcInterpreter.getProperty("zeppelin.jdbc.hive.jobUrl.template");
        boolean jobUrlExtracted = false;
        try {
            while (hiveStmt.hasMoreLogs() && !hiveStmt.isClosed() && !Thread.interrupted()) {
                Thread.sleep(queryInterval);
                List<String> logs = hiveStmt.getQueryLog();
                String logsOutput = StringUtils.join(logs, System.lineSeparator());
                LOGGER.debug("Hive job output: {}", logsOutput);
                boolean displayLogProperty = context.getBooleanLocalProperty("displayLog", displayLog);
                if (!StringUtils.isBlank(logsOutput) && displayLogProperty) {
                    context.out.write(logsOutput + "\n");
                    context.out.flush();
                }
                if (!StringUtils.isBlank(logsOutput) && progressBar != null && displayLogProperty) {
                    progressBar.operationLogShowedToUser();
                }
                if (!jobUrlExtracted) {
                    jobUrlExtracted = extractJobURL(logsOutput, jobUrlTemplate, context);
                }
            }
        } catch (InterruptedException e) {
            LOGGER.warn("Hive monitor thread is interrupted", e);
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            LOGGER.warn("Fail to monitor hive statement", e);
        }
        LOGGER.info("HiveMonitor-Thread is finished");
    });
    thread.setName("HiveMonitor-Thread");
    thread.setDaemon(true);
    thread.start();
    LOGGER.info("Start HiveMonitor-Thread for sql: {}", hiveStmt);
    if (progressBar != null) {
        // old: hiveStmt.setInPlaceUpdateStream(progressBar.getInPlaceUpdateStream(context.out));
        // Move codes into ProgressBar to delay NoClassDefFoundError of InPlaceUpdateStream
        // until ProgressBar instanced.
        // When hive < 2.3, ProgressBar will not be instanced, so it works well.
        progressBar.setInPlaceUpdateStream(hiveStmt, context.out);
    }
}

Example 2 with HiveStatement

use of org.apache.hive.jdbc.HiveStatement in project hive by apache.

the class Commands method showRemainingLogsIfAny.

private void showRemainingLogsIfAny(Statement statement) {
    if (statement instanceof HiveStatement) {
        HiveStatement hiveStatement = (HiveStatement) statement;
        List<String> logs = null;
        do {
            try {
                logs = hiveStatement.getQueryLog();
            } catch (SQLException e) {
                beeLine.error(new SQLWarning(e));
                return;
            }
            for (String log : logs) {
                if (!beeLine.isTestMode()) {
                    beeLine.info(log);
                } else {
                    // In test mode print the logs to the output
                    beeLine.output(log);
                }
            }
        } while (logs.size() > 0);
    } else {
        beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass());
    }
}

Example 3 with HiveStatement

use of org.apache.hive.jdbc.HiveStatement in project hive by apache.

the class Commands method executeInternal.

// Return false only occurred error when execution the sql and the sql should follow the rules
// of beeline.
private boolean executeInternal(String sql, boolean call) {
    if (!beeLine.isBeeLine()) {
        sql = cliToBeelineCmd(sql);
    }
    if (sql == null || sql.length() == 0) {
        return true;
    }
    if (beeLine.isComment(sql)) {
        // skip this and rest cmds in the line
        return true;
    }
    // is source CMD
    if (isSourceCMD(sql)) {
        return sourceFile(sql);
    }
    if (sql.startsWith(BeeLine.COMMAND_PREFIX)) {
        return beeLine.execCommandWithPrefix(sql);
    }
    String prefix = call ? "call" : "sql";
    if (sql.startsWith(prefix)) {
        sql = sql.substring(prefix.length());
    }
    // batch statements?
    if (beeLine.getBatch() != null) {
        beeLine.getBatch().add(sql);
        return true;
    }
    if (!(beeLine.assertConnection())) {
        return false;
    }
    ClientHook hook = ClientCommandHookFactory.get().getHook(beeLine, sql);
    try {
        Statement stmnt = null;
        boolean hasResults;
        Thread logThread = null;
        try {
            long start = System.currentTimeMillis();
            if (call) {
                stmnt = beeLine.getDatabaseConnection().getConnection().prepareCall(sql);
                hasResults = ((CallableStatement) stmnt).execute();
            } else {
                stmnt = beeLine.createStatement();
                // In test mode we want the operation logs regardless of the settings
                if (!beeLine.isTestMode() && beeLine.getOpts().isSilent()) {
                    hasResults = stmnt.execute(sql);
                } else {
                    InPlaceUpdateStream.EventNotifier eventNotifier = new InPlaceUpdateStream.EventNotifier();
                    logThread = new Thread(createLogRunnable(stmnt, eventNotifier));
                    logThread.setDaemon(true);
                    logThread.start();
                    if (stmnt instanceof HiveStatement) {
                        HiveStatement hiveStatement = (HiveStatement) stmnt;
                        hiveStatement.setInPlaceUpdateStream(new BeelineInPlaceUpdateStream(beeLine.getErrorStream(), eventNotifier));
                    }
                    hasResults = stmnt.execute(sql);
                    logThread.interrupt();
                    logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
                }
            }
            beeLine.showWarnings();
            if (hasResults) {
                OutputFile outputFile = beeLine.getRecordOutputFile();
                if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
                    outputFile.fetchStarted();
                    if (!sql.trim().toLowerCase().startsWith("explain")) {
                        outputFile.foundQuery(true);
                    } else {
                        outputFile.foundQuery(false);
                    }
                }
                do {
                    ResultSet rs = stmnt.getResultSet();
                    try {
                        int count = beeLine.print(rs);
                        long end = System.currentTimeMillis();
                        if (showReport()) {
                            beeLine.output(beeLine.loc("rows-selected", count) + " " + beeLine.locElapsedTime(end - start), true, beeLine.getErrorStream());
                        }
                    } finally {
                        if (logThread != null) {
                            logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
                            showRemainingLogsIfAny(stmnt);
                            logThread = null;
                        }
                        rs.close();
                    }
                } while (BeeLine.getMoreResults(stmnt));
                if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
                    outputFile.fetchFinished();
                }
            } else {
                int count = stmnt.getUpdateCount();
                long end = System.currentTimeMillis();
                if (showReport()) {
                    beeLine.output(beeLine.loc("rows-affected", count) + " " + beeLine.locElapsedTime(end - start), true, beeLine.getErrorStream());
                }
            }
        } finally {
            if (logThread != null) {
                if (!logThread.isInterrupted()) {
                    logThread.interrupt();
                }
                logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
                if (stmnt != null) {
                    showRemainingLogsIfAny(stmnt);
                }
            }
            if (stmnt != null) {
                stmnt.close();
            }
        }
    } catch (Exception e) {
        return beeLine.error(e);
    }
    beeLine.showWarnings();
    if (hook != null) {
        hook.postHook(beeLine);
    }
    return true;
}

Example 4 with HiveStatement

use of org.apache.hive.jdbc.HiveStatement in project hive by apache.

the class TestReExecuteKilledTezAMQueryPlugin method testKillQueryById.

@Test
public void testKillQueryById() throws Exception {
    String user = System.getProperty("user.name");
    Connection con1 = getConnection(miniHS2.getJdbcURL(testDbName), user, "bar");
    final HiveStatement stmt = (HiveStatement) con1.createStatement();
    final StringBuffer stmtQueryId = new StringBuffer();
    ExceptionHolder originalQExHolder = new ExceptionHolder();
    originalQExHolder.throwable = null;
    // Thread executing the query
    Thread tExecute = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                System.out.println("Executing query: ");
                stmt.execute("set hive.llap.execution.mode = none");
                // The test table has 500 rows, so total query time should be ~ 500*100ms
                stmt.executeAsync("select sleepMsUDF(t1.int_col, 100), t1.int_col, t2.int_col " + "from " + tableName + " t1 join " + tableName + " t2 on t1.int_col = t2.int_col");
                stmtQueryId.append(stmt.getQueryId());
                stmt.getUpdateCount();
            } catch (SQLException e) {
                originalQExHolder.throwable = e;
            }
        }
    });
    tExecute.start();
    // wait for other thread to create the stmt handle
    int count = 0;
    while (++count <= 10) {
        Thread.sleep(2000);
        if (stmtQueryId.length() != 0) {
            String queryId = stmtQueryId.toString();
            System.out.println("Killing query: " + queryId);
            killAMForQueryId(queryId);
            break;
        }
    }
    tExecute.join();
    try {
        stmt.close();
        con1.close();
    } catch (Exception e) {
        // ignore error
        LOG.warn("Exception when close stmt and con", e);
    }
    Assert.assertEquals(originalQExHolder.throwable, null);
}