use of org.apache.hive.jdbc.HiveStatement in project zeppelin by apache.
the class HiveUtils method startHiveMonitorThread.
/**
* Display hive job execution info, and progress info for hive >= 2.3
*
* @param stmt
* @param context
* @param displayLog
*/
public static void startHiveMonitorThread(Statement stmt, InterpreterContext context, boolean displayLog, JDBCInterpreter jdbcInterpreter) {
HiveStatement hiveStmt = (HiveStatement) ((DelegatingStatement) ((DelegatingStatement) stmt).getDelegate()).getDelegate();
String hiveVersion = HiveVersionInfo.getVersion();
ProgressBar progressBarTemp = null;
if (isProgressBarSupported(hiveVersion)) {
LOGGER.debug("ProgressBar is supported for hive version: {}", hiveVersion);
progressBarTemp = new ProgressBar();
} else {
LOGGER.debug("ProgressBar is not supported for hive version: {}", hiveVersion);
}
// need to use final variable progressBar in thread, so need progressBarTemp here.
final ProgressBar progressBar = progressBarTemp;
final long queryInterval = Long.parseLong(jdbcInterpreter.getProperty("zeppelin.jdbc.hive.monitor.query_interval", DEFAULT_QUERY_PROGRESS_INTERVAL + ""));
Thread thread = new Thread(() -> {
String jobUrlTemplate = jdbcInterpreter.getProperty("zeppelin.jdbc.hive.jobUrl.template");
boolean jobUrlExtracted = false;
try {
while (hiveStmt.hasMoreLogs() && !hiveStmt.isClosed() && !Thread.interrupted()) {
Thread.sleep(queryInterval);
List<String> logs = hiveStmt.getQueryLog();
String logsOutput = StringUtils.join(logs, System.lineSeparator());
LOGGER.debug("Hive job output: {}", logsOutput);
boolean displayLogProperty = context.getBooleanLocalProperty("displayLog", displayLog);
if (!StringUtils.isBlank(logsOutput) && displayLogProperty) {
context.out.write(logsOutput + "\n");
context.out.flush();
}
if (!StringUtils.isBlank(logsOutput) && progressBar != null && displayLogProperty) {
progressBar.operationLogShowedToUser();
}
if (!jobUrlExtracted) {
jobUrlExtracted = extractJobURL(logsOutput, jobUrlTemplate, context);
}
}
} catch (InterruptedException e) {
LOGGER.warn("Hive monitor thread is interrupted", e);
Thread.currentThread().interrupt();
} catch (Exception e) {
LOGGER.warn("Fail to monitor hive statement", e);
}
LOGGER.info("HiveMonitor-Thread is finished");
});
thread.setName("HiveMonitor-Thread");
thread.setDaemon(true);
thread.start();
LOGGER.info("Start HiveMonitor-Thread for sql: {}", hiveStmt);
if (progressBar != null) {
// old: hiveStmt.setInPlaceUpdateStream(progressBar.getInPlaceUpdateStream(context.out));
// Move codes into ProgressBar to delay NoClassDefFoundError of InPlaceUpdateStream
// until ProgressBar instanced.
// When hive < 2.3, ProgressBar will not be instanced, so it works well.
progressBar.setInPlaceUpdateStream(hiveStmt, context.out);
}
}
use of org.apache.hive.jdbc.HiveStatement in project hive by apache.
the class Commands method showRemainingLogsIfAny.
private void showRemainingLogsIfAny(Statement statement) {
if (statement instanceof HiveStatement) {
HiveStatement hiveStatement = (HiveStatement) statement;
List<String> logs = null;
do {
try {
logs = hiveStatement.getQueryLog();
} catch (SQLException e) {
beeLine.error(new SQLWarning(e));
return;
}
for (String log : logs) {
if (!beeLine.isTestMode()) {
beeLine.info(log);
} else {
// In test mode print the logs to the output
beeLine.output(log);
}
}
} while (logs.size() > 0);
} else {
beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass());
}
}
use of org.apache.hive.jdbc.HiveStatement in project hive by apache.
the class Commands method executeInternal.
// Return false only occurred error when execution the sql and the sql should follow the rules
// of beeline.
private boolean executeInternal(String sql, boolean call) {
if (!beeLine.isBeeLine()) {
sql = cliToBeelineCmd(sql);
}
if (sql == null || sql.length() == 0) {
return true;
}
if (beeLine.isComment(sql)) {
// skip this and rest cmds in the line
return true;
}
// is source CMD
if (isSourceCMD(sql)) {
return sourceFile(sql);
}
if (sql.startsWith(BeeLine.COMMAND_PREFIX)) {
return beeLine.execCommandWithPrefix(sql);
}
String prefix = call ? "call" : "sql";
if (sql.startsWith(prefix)) {
sql = sql.substring(prefix.length());
}
// batch statements?
if (beeLine.getBatch() != null) {
beeLine.getBatch().add(sql);
return true;
}
if (!(beeLine.assertConnection())) {
return false;
}
ClientHook hook = ClientCommandHookFactory.get().getHook(beeLine, sql);
try {
Statement stmnt = null;
boolean hasResults;
Thread logThread = null;
try {
long start = System.currentTimeMillis();
if (call) {
stmnt = beeLine.getDatabaseConnection().getConnection().prepareCall(sql);
hasResults = ((CallableStatement) stmnt).execute();
} else {
stmnt = beeLine.createStatement();
// In test mode we want the operation logs regardless of the settings
if (!beeLine.isTestMode() && beeLine.getOpts().isSilent()) {
hasResults = stmnt.execute(sql);
} else {
InPlaceUpdateStream.EventNotifier eventNotifier = new InPlaceUpdateStream.EventNotifier();
logThread = new Thread(createLogRunnable(stmnt, eventNotifier));
logThread.setDaemon(true);
logThread.start();
if (stmnt instanceof HiveStatement) {
HiveStatement hiveStatement = (HiveStatement) stmnt;
hiveStatement.setInPlaceUpdateStream(new BeelineInPlaceUpdateStream(beeLine.getErrorStream(), eventNotifier));
}
hasResults = stmnt.execute(sql);
logThread.interrupt();
logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
}
}
beeLine.showWarnings();
if (hasResults) {
OutputFile outputFile = beeLine.getRecordOutputFile();
if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
outputFile.fetchStarted();
if (!sql.trim().toLowerCase().startsWith("explain")) {
outputFile.foundQuery(true);
} else {
outputFile.foundQuery(false);
}
}
do {
ResultSet rs = stmnt.getResultSet();
try {
int count = beeLine.print(rs);
long end = System.currentTimeMillis();
if (showReport()) {
beeLine.output(beeLine.loc("rows-selected", count) + " " + beeLine.locElapsedTime(end - start), true, beeLine.getErrorStream());
}
} finally {
if (logThread != null) {
logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
showRemainingLogsIfAny(stmnt);
logThread = null;
}
rs.close();
}
} while (BeeLine.getMoreResults(stmnt));
if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
outputFile.fetchFinished();
}
} else {
int count = stmnt.getUpdateCount();
long end = System.currentTimeMillis();
if (showReport()) {
beeLine.output(beeLine.loc("rows-affected", count) + " " + beeLine.locElapsedTime(end - start), true, beeLine.getErrorStream());
}
}
} finally {
if (logThread != null) {
if (!logThread.isInterrupted()) {
logThread.interrupt();
}
logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
if (stmnt != null) {
showRemainingLogsIfAny(stmnt);
}
}
if (stmnt != null) {
stmnt.close();
}
}
} catch (Exception e) {
return beeLine.error(e);
}
beeLine.showWarnings();
if (hook != null) {
hook.postHook(beeLine);
}
return true;
}
use of org.apache.hive.jdbc.HiveStatement in project hive by apache.
the class TestReExecuteKilledTezAMQueryPlugin method testKillQueryById.
@Test
public void testKillQueryById() throws Exception {
String user = System.getProperty("user.name");
Connection con1 = getConnection(miniHS2.getJdbcURL(testDbName), user, "bar");
final HiveStatement stmt = (HiveStatement) con1.createStatement();
final StringBuffer stmtQueryId = new StringBuffer();
ExceptionHolder originalQExHolder = new ExceptionHolder();
originalQExHolder.throwable = null;
// Thread executing the query
Thread tExecute = new Thread(new Runnable() {
@Override
public void run() {
try {
System.out.println("Executing query: ");
stmt.execute("set hive.llap.execution.mode = none");
// The test table has 500 rows, so total query time should be ~ 500*100ms
stmt.executeAsync("select sleepMsUDF(t1.int_col, 100), t1.int_col, t2.int_col " + "from " + tableName + " t1 join " + tableName + " t2 on t1.int_col = t2.int_col");
stmtQueryId.append(stmt.getQueryId());
stmt.getUpdateCount();
} catch (SQLException e) {
originalQExHolder.throwable = e;
}
}
});
tExecute.start();
// wait for other thread to create the stmt handle
int count = 0;
while (++count <= 10) {
Thread.sleep(2000);
if (stmtQueryId.length() != 0) {
String queryId = stmtQueryId.toString();
System.out.println("Killing query: " + queryId);
killAMForQueryId(queryId);
break;
}
}
tExecute.join();
try {
stmt.close();
con1.close();
} catch (Exception e) {
// ignore error
LOG.warn("Exception when close stmt and con", e);
}
Assert.assertEquals(originalQExHolder.throwable, null);
}
Aggregations