Examples with Statement java.sql.Statement used on opensource projects

Example 21 with Statement

use of java.sql.Statement in project hive by apache.

the class CompactionTxnHandler method markFailed.

/**
   * If there is an entry in compaction_queue with ci.id, remove it
   * Make entry in completed_compactions with status 'f'.
   * If there is no entry in compaction_queue, it means Initiator failed to even schedule a compaction,
   * which we record as ATTEMPTED_STATE entry in history.
   */
@Override
@RetrySemantics.CannotRetry
public void markFailed(CompactionInfo ci) throws MetaException {
    //todo: this should take "comment" as parameter to set in CC_META_INFO to provide some context for the failure
    try {
        Connection dbConn = null;
        Statement stmt = null;
        PreparedStatement pStmt = null;
        ResultSet rs = null;
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + ci.id);
            if (rs.next()) {
                ci = CompactionInfo.loadFullFromCompactionQueue(rs);
                String s = "delete from COMPACTION_QUEUE where cq_id = " + ci.id;
                LOG.debug("Going to execute update <" + s + ">");
                int updCnt = stmt.executeUpdate(s);
            } else {
                if (ci.id > 0) {
                    //the record with valid CQ_ID has disappeared - this is a sign of something wrong
                    throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
                }
            }
            if (ci.id == 0) {
                //The failure occurred before we even made an entry in COMPACTION_QUEUE
                //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
                ci.id = generateCompactionQueueId(stmt);
                //mostly this indicates that the Initiator is paying attention to some table even though
                //compactions are not happening.
                ci.state = ATTEMPTED_STATE;
                //this is not strictly accurate, but 'type' cannot be null.
                if (ci.type == null) {
                    ci.type = CompactionType.MINOR;
                }
                ci.start = getDbTime(dbConn);
            } else {
                ci.state = FAILED_STATE;
            }
            close(rs, stmt, null);
            pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
            CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
            int updCount = pStmt.executeUpdate();
            LOG.debug("Going to commit");
            closeStmt(pStmt);
            dbConn.commit();
        } catch (SQLException e) {
            LOG.warn("markFailed(" + ci.id + "):" + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            try {
                checkRetryable(dbConn, e, "markFailed(" + ci + ")");
            } catch (MetaException ex) {
                LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(ex));
            }
            LOG.error("markFailed(" + ci + ") failed: " + e.getMessage(), e);
        } finally {
            close(rs, stmt, null);
            close(null, pStmt, dbConn);
        }
    } catch (RetryException e) {
        markFailed(ci);
    }
}

Example 22 with Statement

use of java.sql.Statement in project hive by apache.

the class CompactionTxnHandler method findPotentialCompactions.

/**
   * This will look through the completed_txn_components table and look for partitions or tables
   * that may be ready for compaction.  Also, look through txns and txn_components tables for
   * aborted transactions that we should add to the list.
   * @param maxAborted Maximum number of aborted queries to allow before marking this as a
   *                   potential compaction.
   * @return list of CompactionInfo structs.  These will not have id, type,
   * or runAs set since these are only potential compactions not actual ones.
   */
@Override
@RetrySemantics.ReadOnly
public Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException {
    Connection dbConn = null;
    Set<CompactionInfo> response = new HashSet<CompactionInfo>();
    Statement stmt = null;
    ResultSet rs = null;
    try {
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            // Check for completed transactions
            String s = "select distinct ctc_database, ctc_table, " + "ctc_partition from COMPLETED_TXN_COMPONENTS";
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            while (rs.next()) {
                CompactionInfo info = new CompactionInfo();
                info.dbname = rs.getString(1);
                info.tableName = rs.getString(2);
                info.partName = rs.getString(3);
                response.add(info);
            }
            rs.close();
            // Check for aborted txns
            s = "select tc_database, tc_table, tc_partition " + "from TXNS, TXN_COMPONENTS " + "where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' " + "group by tc_database, tc_table, tc_partition " + "having count(*) > " + maxAborted;
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            while (rs.next()) {
                CompactionInfo info = new CompactionInfo();
                info.dbname = rs.getString(1);
                info.tableName = rs.getString(2);
                info.partName = rs.getString(3);
                info.tooManyAborts = true;
                response.add(info);
            }
            LOG.debug("Going to rollback");
            dbConn.rollback();
        } catch (SQLException e) {
            LOG.error("Unable to connect to transaction database " + e.getMessage());
            checkRetryable(dbConn, e, "findPotentialCompactions(maxAborted:" + maxAborted + ")");
        } finally {
            close(rs, stmt, dbConn);
        }
        return response;
    } catch (RetryException e) {
        return findPotentialCompactions(maxAborted);
    }
}

Example 23 with Statement

use of java.sql.Statement in project hive by apache.

the class CompactionTxnHandler method findReadyToClean.

/**
   * Find entries in the queue that are ready to
   * be cleaned.
   * @return information on the entry in the queue.
   */
@Override
@RetrySemantics.ReadOnly
public List<CompactionInfo> findReadyToClean() throws MetaException {
    Connection dbConn = null;
    List<CompactionInfo> rc = new ArrayList<CompactionInfo>();
    Statement stmt = null;
    ResultSet rs = null;
    try {
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            String s = "select cq_id, cq_database, cq_table, cq_partition, " + "cq_type, cq_run_as, cq_highest_txn_id from COMPACTION_QUEUE where cq_state = '" + READY_FOR_CLEANING + "'";
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            while (rs.next()) {
                CompactionInfo info = new CompactionInfo();
                info.id = rs.getLong(1);
                info.dbname = rs.getString(2);
                info.tableName = rs.getString(3);
                info.partName = rs.getString(4);
                switch(rs.getString(5).charAt(0)) {
                    case MAJOR_TYPE:
                        info.type = CompactionType.MAJOR;
                        break;
                    case MINOR_TYPE:
                        info.type = CompactionType.MINOR;
                        break;
                    default:
                        throw new MetaException("Unexpected compaction type " + rs.getString(5));
                }
                info.runAs = rs.getString(6);
                info.highestTxnId = rs.getLong(7);
                rc.add(info);
            }
            LOG.debug("Going to rollback");
            dbConn.rollback();
            return rc;
        } catch (SQLException e) {
            LOG.error("Unable to select next element for cleaning, " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "findReadyToClean");
            throw new MetaException("Unable to connect to transaction database " + StringUtils.stringifyException(e));
        } finally {
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        return findReadyToClean();
    }
}

Example 24 with Statement

use of java.sql.Statement in project hive by apache.

the class CompactionTxnHandler method markCleaned.

/**
   * This will remove an entry from the queue after
   * it has been compacted.
   * 
   * @param info info on the compaction entry to remove
   */
@Override
@RetrySemantics.CannotRetry
public void markCleaned(CompactionInfo info) throws MetaException {
    try {
        Connection dbConn = null;
        Statement stmt = null;
        PreparedStatement pStmt = null;
        ResultSet rs = null;
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + info.id);
            if (rs.next()) {
                info = CompactionInfo.loadFullFromCompactionQueue(rs);
            } else {
                throw new IllegalStateException("No record with CQ_ID=" + info.id + " found in COMPACTION_QUEUE");
            }
            close(rs);
            String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id;
            LOG.debug("Going to execute update <" + s + ">");
            int updCount = stmt.executeUpdate(s);
            if (updCount != 1) {
                LOG.error("Unable to delete compaction record: " + info + ".  Update count=" + updCount);
                LOG.debug("Going to rollback");
                dbConn.rollback();
            }
            pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
            info.state = SUCCEEDED_STATE;
            CompactionInfo.insertIntoCompletedCompactions(pStmt, info, getDbTime(dbConn));
            updCount = pStmt.executeUpdate();
            // Remove entries from completed_txn_components as well, so we don't start looking there
            // again but only up to the highest txn ID include in this compaction job.
            //highestTxnId will be NULL in upgrade scenarios
            s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " + "ctc_table = '" + info.tableName + "'";
            if (info.partName != null) {
                s += " and ctc_partition = '" + info.partName + "'";
            }
            if (info.highestTxnId != 0) {
                s += " and ctc_txnid <= " + info.highestTxnId;
            }
            LOG.debug("Going to execute update <" + s + ">");
            if (stmt.executeUpdate(s) < 1) {
                LOG.error("Expected to remove at least one row from completed_txn_components when " + "marking compaction entry as clean!");
            }
            s = "select distinct txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" + info.tableName + "'" + (info.highestTxnId == 0 ? "" : " and txn_id <= " + info.highestTxnId);
            if (info.partName != null)
                s += " and tc_partition = '" + info.partName + "'";
            LOG.debug("Going to execute update <" + s + ">");
            rs = stmt.executeQuery(s);
            List<Long> txnids = new ArrayList<>();
            while (rs.next()) txnids.add(rs.getLong(1));
            // Remove entries from txn_components, as there may be aborted txn components
            if (txnids.size() > 0) {
                List<String> queries = new ArrayList<String>();
                // Prepare prefix and suffix
                StringBuilder prefix = new StringBuilder();
                StringBuilder suffix = new StringBuilder();
                prefix.append("delete from TXN_COMPONENTS where ");
                //because 1 txn may include different partitions/tables even in auto commit mode
                suffix.append(" and tc_database = ");
                suffix.append(quoteString(info.dbname));
                suffix.append(" and tc_table = ");
                suffix.append(quoteString(info.tableName));
                if (info.partName != null) {
                    suffix.append(" and tc_partition = ");
                    suffix.append(quoteString(info.partName));
                }
                // Populate the complete query with provided prefix and suffix
                TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "tc_txnid", true, false);
                for (String query : queries) {
                    LOG.debug("Going to execute update <" + query + ">");
                    int rc = stmt.executeUpdate(query);
                    LOG.debug("Removed " + rc + " records from txn_components");
                // Don't bother cleaning from the txns table.  A separate call will do that.  We don't
                // know here which txns still have components from other tables or partitions in the
                // table, so we don't know which ones we can and cannot clean.
                }
            }
            LOG.debug("Going to commit");
            dbConn.commit();
        } catch (SQLException e) {
            LOG.error("Unable to delete from compaction queue " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "markCleaned(" + info + ")");
            throw new MetaException("Unable to connect to transaction database " + StringUtils.stringifyException(e));
        } finally {
            closeStmt(pStmt);
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        markCleaned(info);
    }
}

Example 25 with Statement

use of java.sql.Statement in project hive by apache.

the class CompactionTxnHandler method checkFailedCompactions.

/**
   * Returns {@code true} if there already exists sufficient number of consecutive failures for
   * this table/partition so that no new automatic compactions will be scheduled.
   * User initiated compactions don't do this check.
   *
   * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
   * That would be a meta operations, i.e. first find all partitions for this table (which have 
   * txn info) and schedule each compaction separately.  This avoids complications in this logic.
   */
@Override
@RetrySemantics.ReadOnly
public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
    Connection dbConn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
        try {
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            rs = stmt.executeQuery("select CC_STATE from COMPLETED_COMPACTIONS where " + "CC_DATABASE = " + quoteString(ci.dbname) + " and " + "CC_TABLE = " + quoteString(ci.tableName) + (ci.partName != null ? "and CC_PARTITION = " + quoteString(ci.partName) : "") + " and CC_STATE != " + quoteChar(ATTEMPTED_STATE) + " order by CC_ID desc");
            int numFailed = 0;
            int numTotal = 0;
            int failedThreshold = conf.getIntVar(HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
            while (rs.next() && ++numTotal <= failedThreshold) {
                if (rs.getString(1).charAt(0) == FAILED_STATE) {
                    numFailed++;
                } else {
                    numFailed--;
                }
            }
            return numFailed == failedThreshold;
        } catch (SQLException e) {
            LOG.error("Unable to delete from compaction queue " + e.getMessage());
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "checkFailedCompactions(" + ci + ")");
            LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(e));
            //weren't able to check
            return false;
        } finally {
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        return checkFailedCompactions(ci);
    }
}