Examples with TxnToWriteId org.apache.hadoop.hive.metastore.api.TxnToWriteId used on opensource projects

Example 1 with TxnToWriteId

use of org.apache.hadoop.hive.metastore.api.TxnToWriteId in project hive by apache.

the class TxnHandler method allocateTableWriteIds.

@Override
public AllocateTableWriteIdsResponse allocateTableWriteIds(AllocateTableWriteIdsRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
    List<Long> txnIds = rqst.getTxnIds();
    String dbName = rqst.getDbName().toLowerCase();
    String tblName = rqst.getTableName().toLowerCase();
    try {
        Connection dbConn = null;
        Statement stmt = null;
        ResultSet rs = null;
        TxnStore.MutexAPI.LockHandle handle = null;
        try {
            lockInternal();
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            // easier to read logs
            Collections.sort(txnIds);
            // Check if all the input txns are in open state. Write ID should be allocated only for open transactions.
            if (!isTxnsInOpenState(txnIds, stmt)) {
                ensureAllTxnsValid(dbName, tblName, txnIds, stmt);
                throw new RuntimeException("This should never happen for txnIds: " + txnIds);
            }
            List<TxnToWriteId> txnToWriteIds = new ArrayList<>();
            List<Long> allocatedTxns = new ArrayList<>();
            long txnId;
            long writeId;
            List<String> queries = new ArrayList<>();
            StringBuilder prefix = new StringBuilder();
            StringBuilder suffix = new StringBuilder();
            // Traverse the TXN_TO_WRITE_ID to see if any of the input txns already have allocated a
            // write id for the same db.table. If yes, then need to reuse it else have to allocate new one
            // The write id would have been already allocated in case of multi-statement txns where
            // first write on a table will allocate write id and rest of the writes should re-use it.
            prefix.append("select t2w_txnid, t2w_writeid from TXN_TO_WRITE_ID where" + " t2w_database = " + quoteString(dbName) + " and t2w_table = " + quoteString(tblName) + " and ");
            suffix.append("");
            TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnIds, "t2w_txnid", false, false);
            for (String query : queries) {
                LOG.debug("Going to execute query <" + query + ">");
                rs = stmt.executeQuery(query);
                while (rs.next()) {
                    // If table write ID is already allocated for the given transaction, then just use it
                    txnId = rs.getLong(1);
                    writeId = rs.getLong(2);
                    txnToWriteIds.add(new TxnToWriteId(txnId, writeId));
                    allocatedTxns.add(txnId);
                    LOG.info("Reused already allocated writeID: " + writeId + " for txnId: " + txnId);
                }
            }
            // If all the txns in the list have already allocated write ids, then just skip new allocations
            long numOfWriteIds = txnIds.size() - allocatedTxns.size();
            assert (numOfWriteIds >= 0);
            if (0 == numOfWriteIds) {
                // If all the txns in the list have pre-allocated write ids for the given table, then just return
                return new AllocateTableWriteIdsResponse(txnToWriteIds);
            }
            handle = getMutexAPI().acquireLock(MUTEX_KEY.WriteIdAllocator.name());
            // There are some txns in the list which has no write id allocated and hence go ahead and do it.
            // Get the next write id for the given table and update it with new next write id.
            // This is select for update query which takes a lock if the table entry is already there in NEXT_WRITE_ID
            String s = sqlGenerator.addForUpdateClause("select nwi_next from NEXT_WRITE_ID where nwi_database = " + quoteString(dbName) + " and nwi_table = " + quoteString(tblName));
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            if (!rs.next()) {
                // First allocation of write id should add the table to the next_write_id meta table
                // The initial value for write id should be 1 and hence we add 1 with number of write ids allocated here
                s = "insert into NEXT_WRITE_ID (nwi_database, nwi_table, nwi_next) values (" + quoteString(dbName) + "," + quoteString(tblName) + "," + String.valueOf(numOfWriteIds + 1) + ")";
                LOG.debug("Going to execute insert <" + s + ">");
                stmt.execute(s);
                writeId = 1;
            } else {
                // Update the NEXT_WRITE_ID for the given table after incrementing by number of write ids allocated
                writeId = rs.getLong(1);
                s = "update NEXT_WRITE_ID set nwi_next = " + (writeId + numOfWriteIds) + " where nwi_database = " + quoteString(dbName) + " and nwi_table = " + quoteString(tblName);
                LOG.debug("Going to execute update <" + s + ">");
                stmt.executeUpdate(s);
            }
            // Map the newly allocated write ids against the list of txns which doesn't have pre-allocated
            // write ids
            List<String> rows = new ArrayList<>();
            for (long txn : txnIds) {
                if (allocatedTxns.contains(txn)) {
                    continue;
                }
                rows.add(txn + ", " + quoteString(dbName) + ", " + quoteString(tblName) + ", " + writeId);
                txnToWriteIds.add(new TxnToWriteId(txn, writeId));
                LOG.info("Allocated writeID: " + writeId + " for txnId: " + txn);
                writeId++;
            }
            // Insert entries to TXN_TO_WRITE_ID for newly allocated write ids
            List<String> inserts = sqlGenerator.createInsertValuesStmt("TXN_TO_WRITE_ID (t2w_txnid, t2w_database, t2w_table, t2w_writeid)", rows);
            for (String insert : inserts) {
                LOG.debug("Going to execute insert <" + insert + ">");
                stmt.execute(insert);
            }
            LOG.debug("Going to commit");
            dbConn.commit();
            return new AllocateTableWriteIdsResponse(txnToWriteIds);
        } catch (SQLException e) {
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "allocateTableWriteIds(" + rqst + ")");
            throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e));
        } finally {
            close(rs, stmt, dbConn);
            if (handle != null) {
                handle.releaseLocks();
            }
            unlockInternal();
        }
    } catch (RetryException e) {
        return allocateTableWriteIds(rqst);
    }
}