Examples with Connection java.sql.Connection used on opensource projects

Example 31 with Connection

use of java.sql.Connection in project hive by apache.

the class CompactionTxnHandler method markCleaned.

/**
   * This will remove an entry from the queue after
   * it has been compacted.
   * 
   * @param info info on the compaction entry to remove
   */
@Override
@RetrySemantics.CannotRetry
public void markCleaned(CompactionInfo info) throws MetaException {
    try {
        Connection dbConn = null;
        Statement stmt = null;
        PreparedStatement pStmt = null;
        ResultSet rs = null;
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + info.id);
            if (rs.next()) {
                info = CompactionInfo.loadFullFromCompactionQueue(rs);
            } else {
                throw new IllegalStateException("No record with CQ_ID=" + info.id + " found in COMPACTION_QUEUE");
            }
            close(rs);
            String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id;
            LOG.debug("Going to execute update <" + s + ">");
            int updCount = stmt.executeUpdate(s);
            if (updCount != 1) {
                LOG.error("Unable to delete compaction record: " + info + ".  Update count=" + updCount);
                LOG.debug("Going to rollback");
                dbConn.rollback();
            }
            pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
            info.state = SUCCEEDED_STATE;
            CompactionInfo.insertIntoCompletedCompactions(pStmt, info, getDbTime(dbConn));
            updCount = pStmt.executeUpdate();
            // Remove entries from completed_txn_components as well, so we don't start looking there
            // again but only up to the highest txn ID include in this compaction job.
            //highestTxnId will be NULL in upgrade scenarios
            s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " + "ctc_table = '" + info.tableName + "'";
            if (info.partName != null) {
                s += " and ctc_partition = '" + info.partName + "'";
            }
            if (info.highestTxnId != 0) {
                s += " and ctc_txnid <= " + info.highestTxnId;
            }
            LOG.debug("Going to execute update <" + s + ">");
            if (stmt.executeUpdate(s) < 1) {
                LOG.error("Expected to remove at least one row from completed_txn_components when " + "marking compaction entry as clean!");
            }
            s = "select distinct txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" + info.tableName + "'" + (info.highestTxnId == 0 ? "" : " and txn_id <= " + info.highestTxnId);
            if (info.partName != null)
                s += " and tc_partition = '" + info.partName + "'";
            LOG.debug("Going to execute update <" + s + ">");
            rs = stmt.executeQuery(s);
            List<Long> txnids = new ArrayList<>();
            while (rs.next()) txnids.add(rs.getLong(1));
            // Remove entries from txn_components, as there may be aborted txn components
            if (txnids.size() > 0) {
                List<String> queries = new ArrayList<String>();
                // Prepare prefix and suffix
                StringBuilder prefix = new StringBuilder();
                StringBuilder suffix = new StringBuilder();
                prefix.append("delete from TXN_COMPONENTS where ");
                //because 1 txn may include different partitions/tables even in auto commit mode
                suffix.append(" and tc_database = ");
                suffix.append(quoteString(info.dbname));
                suffix.append(" and tc_table = ");
                suffix.append(quoteString(info.tableName));
                if (info.partName != null) {
                    suffix.append(" and tc_partition = ");
                    suffix.append(quoteString(info.partName));
                }
                // Populate the complete query with provided prefix and suffix
                TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "tc_txnid", true, false);
                for (String query : queries) {
                    LOG.debug("Going to execute update <" + query + ">");
                    int rc = stmt.executeUpdate(query);
                    LOG.debug("Removed " + rc + " records from txn_components");
                // Don't bother cleaning from the txns table.  A separate call will do that.  We don't
                // know here which txns still have components from other tables or partitions in the
                // table, so we don't know which ones we can and cannot clean.
                }
            }
            LOG.debug("Going to commit");
            dbConn.commit();
        } catch (SQLException e) {
            LOG.error("Unable to delete from compaction queue " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "markCleaned(" + info + ")");
            throw new MetaException("Unable to connect to transaction database " + StringUtils.stringifyException(e));
        } finally {
            closeStmt(pStmt);
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        markCleaned(info);
    }
}

Example 32 with Connection

use of java.sql.Connection in project hive by apache.

the class CompactionTxnHandler method checkFailedCompactions.

/**
   * Returns {@code true} if there already exists sufficient number of consecutive failures for
   * this table/partition so that no new automatic compactions will be scheduled.
   * User initiated compactions don't do this check.
   *
   * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
   * That would be a meta operations, i.e. first find all partitions for this table (which have 
   * txn info) and schedule each compaction separately.  This avoids complications in this logic.
   */
@Override
@RetrySemantics.ReadOnly
public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
    Connection dbConn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            rs = stmt.executeQuery("select CC_STATE from COMPLETED_COMPACTIONS where " + "CC_DATABASE = " + quoteString(ci.dbname) + " and " + "CC_TABLE = " + quoteString(ci.tableName) + (ci.partName != null ? "and CC_PARTITION = " + quoteString(ci.partName) : "") + " and CC_STATE != " + quoteChar(ATTEMPTED_STATE) + " order by CC_ID desc");
            int numFailed = 0;
            int numTotal = 0;
            int failedThreshold = conf.getIntVar(HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
            while (rs.next() && ++numTotal <= failedThreshold) {
                if (rs.getString(1).charAt(0) == FAILED_STATE) {
                    numFailed++;
                } else {
                    numFailed--;
                }
            }
            return numFailed == failedThreshold;
        } catch (SQLException e) {
            LOG.error("Unable to delete from compaction queue " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "checkFailedCompactions(" + ci + ")");
            LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(e));
            //weren't able to check
            return false;
        } finally {
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        return checkFailedCompactions(ci);
    }
}

Example 33 with Connection

use of java.sql.Connection in project hive by apache.

the class CompactionTxnHandler method cleanEmptyAbortedTxns.

/**
   * Clean up aborted transactions from txns that have no components in txn_components.  The reason such
   * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
   * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
   */
@Override
@RetrySemantics.SafeToRetry
public void cleanEmptyAbortedTxns() throws MetaException {
    try {
        Connection dbConn = null;
        Statement stmt = null;
        ResultSet rs = null;
        try {
            //Aborted is a terminal state, so nothing about the txn can change
            //after that, so READ COMMITTED is sufficient.
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            String s = "select txn_id from TXNS where " + "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " + "txn_state = '" + TXN_ABORTED + "'";
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            List<Long> txnids = new ArrayList<>();
            while (rs.next()) txnids.add(rs.getLong(1));
            close(rs);
            if (txnids.size() <= 0) {
                return;
            }
            //easier to read logs
            Collections.sort(txnids);
            List<String> queries = new ArrayList<String>();
            StringBuilder prefix = new StringBuilder();
            StringBuilder suffix = new StringBuilder();
            prefix.append("delete from TXNS where ");
            suffix.append("");
            TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "txn_id", false, false);
            for (String query : queries) {
                LOG.debug("Going to execute update <" + query + ">");
                int rc = stmt.executeUpdate(query);
                LOG.info("Removed " + rc + "  empty Aborted transactions from TXNS");
            }
            LOG.info("Aborted transactions removed from TXNS: " + txnids);
            LOG.debug("Going to commit");
            dbConn.commit();
        } catch (SQLException e) {
            LOG.error("Unable to delete from txns table " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "cleanEmptyAbortedTxns");
            throw new MetaException("Unable to connect to transaction database " + StringUtils.stringifyException(e));
        } finally {
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        cleanEmptyAbortedTxns();
    }
}

Example 34 with Connection

use of java.sql.Connection in project hive by apache.

the class TestTxnUtils method runAgainstDerby.

/** Verify queries can run against Derby DB.
   *  As long as Derby doesn't complain, we assume the query is syntactically/semantically correct.
   */
private void runAgainstDerby(List<String> queries) throws Exception {
    Connection conn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
        conn = TxnDbUtil.getConnection();
        stmt = conn.createStatement();
        for (String query : queries) {
            rs = stmt.executeQuery(query);
            Assert.assertTrue("The query is not valid", rs.next());
        }
    } finally {
        TxnDbUtil.closeResources(conn, stmt, rs);
    }
}

Example 35 with Connection

use of java.sql.Connection in project hive by apache.

the class ThriftCliServiceMessageSizeTest method testMessageSize.

@Test
public void testMessageSize() throws Exception {
    String transportMode = "binary";
    hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false);
    hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host);
    hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port);
    hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NONE.toString());
    hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode);
    HiveServer2 hiveServer2 = new HiveServer2();
    String url = "jdbc:hive2://localhost:" + port + "/default";
    Class.forName("org.apache.hive.jdbc.HiveDriver");
    try {
        // First start HS2 with high message size limit. This should allow connections
        hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 100 * 1024 * 1024);
        startHiveServer2WithConf(hiveServer2, hiveConf);
        System.out.println("Started Thrift CLI service with message size limit " + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE));
        // With the high message size limit this connection should work
        Connection connection = DriverManager.getConnection(url, "hiveuser", "hive");
        Statement stmt = connection.createStatement();
        assertNotNull("Statement is null", stmt);
        stmt.execute("set hive.support.concurrency = false");
        connection.close();
        stopHiveServer2(hiveServer2);
        // Now start HS2 with low message size limit. This should prevent any connections
        hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 1);
        hiveServer2 = new HiveServer2();
        startHiveServer2WithConf(hiveServer2, hiveConf);
        System.out.println("Started Thrift CLI service with message size limit " + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE));
        Exception caughtException = null;
        try {
            // This should fail
            connection = DriverManager.getConnection(url, "hiveuser", "hive");
        } catch (Exception err) {
            caughtException = err;
        }
        // Verify we hit an error while connecting
        assertNotNull(caughtException);
    } finally {
        stopHiveServer2(hiveServer2);
        hiveServer2 = null;
    }
}