use of org.apache.hadoop.hive.metastore.api.LockType in project hive by apache.
the class TxnHandler method showLocks.
@RetrySemantics.ReadOnly
public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException {
try {
Connection dbConn = null;
ShowLocksResponse rsp = new ShowLocksResponse();
List<ShowLocksResponseElement> elems = new ArrayList<>();
List<LockInfoExt> sortedList = new ArrayList<>();
PreparedStatement pst = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
String s = "SELECT \"HL_LOCK_EXT_ID\", \"HL_TXNID\", \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\", \"HL_LOCK_STATE\", " + "\"HL_LOCK_TYPE\", \"HL_LAST_HEARTBEAT\", \"HL_ACQUIRED_AT\", \"HL_USER\", \"HL_HOST\", \"HL_LOCK_INT_ID\"," + "\"HL_BLOCKEDBY_EXT_ID\", \"HL_BLOCKEDBY_INT_ID\", \"HL_AGENT_INFO\" FROM \"HIVE_LOCKS\"";
// Some filters may have been specified in the SHOW LOCKS statement. Add them to the query.
String dbName = rqst.getDbname();
String tableName = rqst.getTablename();
String partName = rqst.getPartname();
List<String> params = new ArrayList<>();
StringBuilder filter = new StringBuilder();
if (dbName != null && !dbName.isEmpty()) {
filter.append("\"HL_DB\"=?");
params.add(dbName);
}
if (tableName != null && !tableName.isEmpty()) {
if (filter.length() > 0) {
filter.append(" and ");
}
filter.append("\"HL_TABLE\"=?");
params.add(tableName);
}
if (partName != null && !partName.isEmpty()) {
if (filter.length() > 0) {
filter.append(" and ");
}
filter.append("\"HL_PARTITION\"=?");
params.add(partName);
}
if (rqst.isSetTxnid()) {
if (filter.length() > 0) {
filter.append(" and ");
}
filter.append("\"HL_TXNID\"=" + rqst.getTxnid());
}
String whereClause = filter.toString();
if (!whereClause.isEmpty()) {
s = s + " where " + whereClause;
}
pst = sqlGenerator.prepareStmtWithParameters(dbConn, s, params);
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = pst.executeQuery();
while (rs.next()) {
ShowLocksResponseElement e = new ShowLocksResponseElement();
e.setLockid(rs.getLong(1));
long txnid = rs.getLong(2);
if (!rs.wasNull())
e.setTxnid(txnid);
e.setDbname(rs.getString(3));
e.setTablename(rs.getString(4));
String partition = rs.getString(5);
if (partition != null)
e.setPartname(partition);
switch(rs.getString(6).charAt(0)) {
case LOCK_ACQUIRED:
e.setState(LockState.ACQUIRED);
break;
case LOCK_WAITING:
e.setState(LockState.WAITING);
break;
default:
throw new MetaException("Unknown lock state " + rs.getString(6).charAt(0));
}
char lockChar = rs.getString(7).charAt(0);
LockType lockType = LockTypeUtil.getLockTypeFromEncoding(lockChar).orElseThrow(() -> new MetaException("Unknown lock type: " + lockChar));
e.setType(lockType);
e.setLastheartbeat(rs.getLong(8));
long acquiredAt = rs.getLong(9);
if (!rs.wasNull())
e.setAcquiredat(acquiredAt);
e.setUser(rs.getString(10));
e.setHostname(rs.getString(11));
e.setLockIdInternal(rs.getLong(12));
long id = rs.getLong(13);
if (!rs.wasNull()) {
e.setBlockedByExtId(id);
}
id = rs.getLong(14);
if (!rs.wasNull()) {
e.setBlockedByIntId(id);
}
e.setAgentInfo(rs.getString(15));
sortedList.add(new LockInfoExt(e));
}
} catch (SQLException e) {
checkRetryable(e, "showLocks(" + rqst + ")");
throw new MetaException("Unable to select from transaction database " + StringUtils.stringifyException(e));
} finally {
closeStmt(pst);
closeDbConn(dbConn);
}
// this ensures that "SHOW LOCKS" prints the locks in the same order as they are examined
// by checkLock() - makes diagnostics easier.
Collections.sort(sortedList, new LockInfoComparator());
for (LockInfoExt lockInfoExt : sortedList) {
elems.add(lockInfoExt.e);
}
rsp.setLocks(elems);
return rsp;
} catch (RetryException e) {
return showLocks(rqst);
}
}
use of org.apache.hadoop.hive.metastore.api.LockType in project hive by apache.
the class TxnHandler method checkLock.
/**
* Lock acquisition is meant to be fair, so every lock can only block on some lock with smaller
* hl_lock_ext_id by only checking earlier locks.
*
* For any given SQL statement all locks required by it are grouped under single extLockId and are
* granted all at once or all locks wait.
*
* This is expected to run at READ_COMMITTED.
*
* If there is a concurrent commitTxn/rollbackTxn, those can only remove rows from HIVE_LOCKS.
* If they happen to be for the same txnid, there will be a WW conflict (in MS DB), if different txnid,
* checkLock() will in the worst case keep locks in Waiting state a little longer.
*/
@RetrySemantics.SafeToRetry("See @SafeToRetry")
private LockResponse checkLock(Connection dbConn, long extLockId, long txnId, boolean zeroWaitReadEnabled) throws NoSuchLockException, TxnAbortedException, MetaException, SQLException {
Statement stmt = null;
ResultSet rs = null;
LockResponse response = new LockResponse();
/**
* todo: Longer term we should pass this from client somehow - this would be an optimization; once
* that is in place make sure to build and test "writeSet" below using OperationType not LockType
* With Static Partitions we assume that the query modifies exactly the partitions it locked. (not entirely
* realistic since Update/Delete may have some predicate that filters out all records out of
* some partition(s), but plausible). For DP, we acquire locks very wide (all known partitions),
* but for most queries only a fraction will actually be updated. #addDynamicPartitions() tells
* us exactly which ones were written to. Thus using this trick to kill a query early for
* DP queries may be too restrictive.
*/
boolean isPartOfDynamicPartitionInsert = true;
try {
// being acquired now
List<LockInfo> locksBeingChecked = getLocksFromLockId(dbConn, extLockId);
response.setLockid(extLockId);
// This is the set of entities that the statement represented by extLockId wants to update
List<LockInfo> writeSet = new ArrayList<>();
for (LockInfo info : locksBeingChecked) {
if (!isPartOfDynamicPartitionInsert && info.type == LockType.SHARED_WRITE) {
writeSet.add(info);
}
}
if (!writeSet.isEmpty()) {
if (writeSet.get(0).txnId == 0) {
// Write operation always start a txn
throw new IllegalStateException("Found Write lock for " + JavaUtils.lockIdToString(extLockId) + " but no txnid");
}
stmt = dbConn.createStatement();
StringBuilder sb = new StringBuilder(" \"WS_DATABASE\", \"WS_TABLE\", \"WS_PARTITION\", " + "\"WS_TXNID\", \"WS_COMMIT_ID\" " + "FROM \"WRITE_SET\" WHERE WS_COMMIT_ID >= " + writeSet.get(0).txnId + // see commitTxn() for more info on this inequality
" AND (");
for (LockInfo info : writeSet) {
sb.append("(\"WS_DATABASE\" = ").append(quoteString(info.db)).append(" AND \"WS_TABLE\" = ").append(quoteString(info.table)).append(" AND \"WS_PARTITION\" ").append(info.partition == null ? "IS NULL" : "= " + quoteString(info.partition)).append(") OR ");
}
// nuke trailing " or "
sb.setLength(sb.length() - 4);
sb.append(")");
// 1 row is sufficient to know we have to kill the query
rs = stmt.executeQuery(sqlGenerator.addLimitClause(1, sb.toString()));
if (rs.next()) {
/**
* if here, it means we found an already committed txn which overlaps with the current one and
* it updated the same resource the current txn wants to update. By First-committer-wins
* rule, current txn will not be allowed to commit so may as well kill it now; This is just an
* optimization to prevent wasting cluster resources to run a query which is known to be DOA.
* {@link #commitTxn(CommitTxnRequest)} has the primary responsibility to ensure this.
* checkLock() runs at READ_COMMITTED so you could have another (Hive) txn running commitTxn()
* in parallel and thus writing to WRITE_SET. commitTxn() logic is properly mutexed to ensure
* that we don't "miss" any WW conflicts. We could've mutexed the checkLock() and commitTxn()
* as well but this reduces concurrency for very little gain.
* Note that update/delete (which runs as dynamic partition insert) acquires a lock on the table,
* but WRITE_SET has entries for actual partitions updated. Thus this optimization will "miss"
* the WW conflict but it will be caught in commitTxn() where actual partitions written are known.
* This is OK since we want 2 concurrent updates that update different sets of partitions to both commit.
*/
String resourceName = rs.getString(1) + '/' + rs.getString(2);
String partName = rs.getString(3);
if (partName != null) {
resourceName += '/' + partName;
}
String msg = "Aborting " + JavaUtils.txnIdToString(writeSet.get(0).txnId) + " since a concurrent committed transaction [" + JavaUtils.txnIdToString(rs.getLong(4)) + "," + rs.getLong(5) + "] has already updated resource '" + resourceName + "'";
LOG.info(msg);
if (abortTxns(dbConn, Collections.singletonList(writeSet.get(0).txnId), false, false) != 1) {
throw new IllegalStateException(msg + " FAILED!");
}
dbConn.commit();
throw new TxnAbortedException(msg);
}
close(rs, stmt, null);
}
String queryStr = " \"EX\".*, \"REQ\".\"HL_LOCK_INT_ID\" \"LOCK_INT_ID\", \"REQ\".\"HL_LOCK_TYPE\" \"LOCK_TYPE\" FROM (" + " SELECT \"HL_LOCK_EXT_ID\", \"HL_LOCK_INT_ID\", \"HL_TXNID\", \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\"," + " \"HL_LOCK_STATE\", \"HL_LOCK_TYPE\" FROM \"HIVE_LOCKS\"" + " WHERE \"HL_LOCK_EXT_ID\" < " + extLockId + ") \"EX\"" + " INNER JOIN (" + " SELECT \"HL_LOCK_INT_ID\", \"HL_TXNID\", \"HL_DB\", \"HL_TABLE\", \"HL_PARTITION\"," + " \"HL_LOCK_TYPE\" FROM \"HIVE_LOCKS\"" + " WHERE \"HL_LOCK_EXT_ID\" = " + extLockId + ") \"REQ\"" + " ON \"EX\".\"HL_DB\" = \"REQ\".\"HL_DB\"" + " AND (\"EX\".\"HL_TABLE\" IS NULL OR \"REQ\".\"HL_TABLE\" IS NULL" + " OR \"EX\".\"HL_TABLE\" = \"REQ\".\"HL_TABLE\"" + " AND (\"EX\".\"HL_PARTITION\" IS NULL OR \"REQ\".\"HL_PARTITION\" IS NULL" + " OR \"EX\".\"HL_PARTITION\" = \"REQ\".\"HL_PARTITION\"))" + /*different locks from same txn should not conflict with each other,
txnId=0 means it's a select or IUD which does not write to ACID table*/
" WHERE (\"REQ\".\"HL_TXNID\" = 0 OR \"EX\".\"HL_TXNID\" != \"REQ\".\"HL_TXNID\")" + " AND ";
/*EXCLUSIVE lock on partition should prevent SHARED_READ on the table, however there is no reason
for an EXCLUSIVE on a table to prevent SHARED_READ on a database. Similarly, EXCLUSIVE on a partition
should not conflict with SHARED_READ on a database.
SHARED_READ is usually acquired on a database to make sure it's not dropped, while some operation
is performed on that db (e.g. show tables, created table, etc).
EXCLUSIVE on an object may mean it's being dropped or overwritten.*/
String[] whereStr = { // shared-read
" \"REQ\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.sharedRead() + " AND \"EX\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.exclusive() + " AND NOT (\"EX\".\"HL_TABLE\" IS NOT NULL AND \"REQ\".\"HL_TABLE\" IS NULL)", // exclusive
" \"REQ\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.exclusive() + " AND NOT (\"EX\".\"HL_TABLE\" IS NULL AND \"EX\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.sharedRead() + " AND \"REQ\".\"HL_TABLE\" IS NOT NULL)", // shared-write
" \"REQ\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.sharedWrite() + " AND \"EX\".\"HL_LOCK_TYPE\" IN (" + LockTypeUtil.exclWrite() + "," + LockTypeUtil.exclusive() + ")", // excl-write
" \"REQ\".\"HL_LOCK_TYPE\"=" + LockTypeUtil.exclWrite() + " AND \"EX\".\"HL_LOCK_TYPE\"!=" + LockTypeUtil.sharedRead() };
List<String> subQuery = new ArrayList<>();
for (String subCond : whereStr) {
subQuery.add("(" + sqlGenerator.addLimitClause(1, queryStr + subCond) + ")");
}
String query = String.join(" UNION ALL ", subQuery);
stmt = dbConn.createStatement();
LOG.debug("Going to execute query <" + query + ">");
rs = stmt.executeQuery(query);
if (rs.next()) {
// We acquire all locks for a given query atomically; if 1 blocks, all remain in Waiting state.
LockInfo blockedBy = new LockInfo(rs);
long intLockId = rs.getLong("LOCK_INT_ID");
char lockChar = rs.getString("LOCK_TYPE").charAt(0);
LOG.debug("Failure to acquire lock({} intLockId:{} {}), blocked by ({})", JavaUtils.lockIdToString(extLockId), intLockId, JavaUtils.txnIdToString(txnId), blockedBy);
if (zeroWaitReadEnabled && isValidTxn(txnId)) {
LockType lockType = LockTypeUtil.getLockTypeFromEncoding(lockChar).orElseThrow(() -> new MetaException("Unknown lock type: " + lockChar));
if (lockType == LockType.SHARED_READ) {
String cleanupQuery = "DELETE FROM \"HIVE_LOCKS\" WHERE \"HL_LOCK_EXT_ID\" = " + extLockId;
LOG.debug("Going to execute query: <" + cleanupQuery + ">");
stmt.executeUpdate(cleanupQuery);
dbConn.commit();
response.setErrorMessage(String.format("Unable to acquire read lock due to an exclusive lock {%s}", blockedBy));
response.setState(LockState.NOT_ACQUIRED);
return response;
}
}
String updateBlockedByQuery = "UPDATE \"HIVE_LOCKS\"" + " SET \"HL_BLOCKEDBY_EXT_ID\" = " + blockedBy.extLockId + ", \"HL_BLOCKEDBY_INT_ID\" = " + blockedBy.intLockId + " WHERE \"HL_LOCK_EXT_ID\" = " + extLockId + " AND \"HL_LOCK_INT_ID\" = " + intLockId;
LOG.debug("Going to execute query: <" + updateBlockedByQuery + ">");
int updCnt = stmt.executeUpdate(updateBlockedByQuery);
if (updCnt != 1) {
LOG.error("Failure to update lock (extLockId={}, intLockId={}) with the blocking lock's IDs " + "(extLockId={}, intLockId={})", extLockId, intLockId, blockedBy.extLockId, blockedBy.intLockId);
shouldNeverHappen(txnId, extLockId, intLockId);
}
dbConn.commit();
response.setState(LockState.WAITING);
return response;
}
// If here, there were no locks that would block any item from 'locksBeingChecked' - acquire them all
acquire(dbConn, stmt, locksBeingChecked);
// We acquired all the locks, so commit and return acquired.
LOG.debug("Successfully acquired locks: " + locksBeingChecked);
dbConn.commit();
response.setState(LockState.ACQUIRED);
} finally {
close(rs, stmt, null);
}
return response;
}
use of org.apache.hadoop.hive.metastore.api.LockType in project hive by apache.
the class AcidUtils method makeLockComponents.
/**
* Create lock components from write/read entities.
* @param outputs write entities
* @param inputs read entities
* @param conf
* @return list with lock components
*/
public static List<LockComponent> makeLockComponents(Set<WriteEntity> outputs, Set<ReadEntity> inputs, Context.Operation operation, HiveConf conf) {
List<LockComponent> lockComponents = new ArrayList<>();
boolean skipReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_READ_LOCKS);
boolean skipNonAcidReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_NONACID_READ_LOCKS);
boolean sharedWrite = !conf.getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK);
boolean isMerge = operation == Context.Operation.MERGE;
// We don't want to acquire read locks during update or delete as we'll be acquiring write
// locks instead. Also, there's no need to lock temp tables since they're session wide
List<ReadEntity> readEntities = inputs.stream().filter(input -> !input.isDummy() && input.needsLock() && !input.isUpdateOrDelete() && AcidUtils.needsLock(input) && !skipReadLock).collect(Collectors.toList());
Set<Table> fullTableLock = getFullTableLock(readEntities, conf);
// For each source to read, get a shared_read lock
for (ReadEntity input : readEntities) {
LockComponentBuilder compBuilder = new LockComponentBuilder();
compBuilder.setSharedRead();
compBuilder.setOperationType(DataOperationType.SELECT);
Table t = null;
switch(input.getType()) {
case DATABASE:
compBuilder.setDbName(input.getDatabase().getName());
break;
case TABLE:
t = input.getTable();
if (!fullTableLock.contains(t)) {
continue;
}
compBuilder.setDbName(t.getDbName());
compBuilder.setTableName(t.getTableName());
break;
case PARTITION:
case DUMMYPARTITION:
compBuilder.setPartitionName(input.getPartition().getName());
t = input.getPartition().getTable();
if (fullTableLock.contains(t)) {
continue;
}
compBuilder.setDbName(t.getDbName());
compBuilder.setTableName(t.getTableName());
break;
default:
// This is a file or something we don't hold locks for.
continue;
}
if (skipNonAcidReadLock && !AcidUtils.isTransactionalTable(t)) {
// read-locks don't protect non-transactional tables data consistency
continue;
}
if (t != null) {
compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t));
}
LockComponent comp = compBuilder.build();
LOG.debug("Adding lock component to lock request {} ", comp);
lockComponents.add(comp);
}
// need a SHARED_WRITE.
for (WriteEntity output : outputs) {
LOG.debug("output is null " + (output == null));
if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR || !AcidUtils.needsLock(output)) {
// We don't lock files or directories. We also skip locking temp tables.
continue;
}
LockComponentBuilder compBuilder = new LockComponentBuilder();
Table t = null;
/**
* For any insert/updates set dir cache to read-only mode, where it wouldn't
* add any new entry to cache.
* When updates are executed, delta folders are created only at the end of the statement
* and at the time of acquiring locks, there would not be any delta folders. This can cause wrong data to be reported
* when "insert" followed by "update" statements are executed. In such cases, use the cache as read only mode.
*/
HiveConf.setIntVar(conf, ConfVars.HIVE_TXN_ACID_DIR_CACHE_DURATION, 0);
switch(output.getType()) {
case DATABASE:
compBuilder.setDbName(output.getDatabase().getName());
break;
case TABLE:
case // in case of dynamic partitioning lock the table
DUMMYPARTITION:
t = output.getTable();
compBuilder.setDbName(t.getDbName());
compBuilder.setTableName(t.getTableName());
break;
case PARTITION:
compBuilder.setPartitionName(output.getPartition().getName());
t = output.getPartition().getTable();
compBuilder.setDbName(t.getDbName());
compBuilder.setTableName(t.getTableName());
break;
default:
// This is a file or something we don't hold locks for.
continue;
}
switch(output.getWriteType()) {
/* base this on HiveOperation instead? this and DDL_NO_LOCK is peppered all over the code...
Seems much cleaner if each stmt is identified as a particular HiveOperation (which I'd think
makes sense everywhere). This however would be problematic for merge...*/
case DDL_EXCLUSIVE:
compBuilder.setExclusive();
compBuilder.setOperationType(DataOperationType.NO_TXN);
break;
case DDL_EXCL_WRITE:
compBuilder.setExclWrite();
compBuilder.setOperationType(DataOperationType.NO_TXN);
break;
case INSERT_OVERWRITE:
t = AcidUtils.getTable(output);
if (AcidUtils.isTransactionalTable(t)) {
if (conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK) && !sharedWrite) {
compBuilder.setExclusive();
} else {
compBuilder.setExclWrite();
}
compBuilder.setOperationType(DataOperationType.UPDATE);
} else {
compBuilder.setExclusive();
compBuilder.setOperationType(DataOperationType.NO_TXN);
}
break;
case INSERT:
assert t != null;
if (AcidUtils.isTransactionalTable(t)) {
boolean isExclMergeInsert = conf.getBoolVar(ConfVars.TXN_MERGE_INSERT_X_LOCK) && isMerge;
if (sharedWrite) {
compBuilder.setSharedWrite();
} else {
if (isExclMergeInsert) {
compBuilder.setExclWrite();
} else {
compBuilder.setSharedRead();
}
}
if (isExclMergeInsert) {
compBuilder.setOperationType(DataOperationType.UPDATE);
break;
}
} else if (MetaStoreUtils.isNonNativeTable(t.getTTable())) {
final HiveStorageHandler storageHandler = Preconditions.checkNotNull(t.getStorageHandler(), "Thought all the non native tables have an instance of storage handler");
LockType lockType = storageHandler.getLockType(output);
if (null == LockType.findByValue(lockType.getValue())) {
throw new IllegalArgumentException(String.format("Lock type [%s] for Database.Table [%s.%s] is unknown", lockType, t.getDbName(), t.getTableName()));
}
compBuilder.setLock(lockType);
} else {
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
compBuilder.setExclusive();
} else {
// this is backward compatible for non-ACID resources, w/o ACID semantics
compBuilder.setSharedRead();
}
}
compBuilder.setOperationType(DataOperationType.INSERT);
break;
case DDL_SHARED:
compBuilder.setSharedRead();
if (output.isTxnAnalyze()) {
// still present.
continue;
}
compBuilder.setOperationType(DataOperationType.NO_TXN);
break;
case UPDATE:
case DELETE:
assert t != null;
if (AcidUtils.isTransactionalTable(t) && sharedWrite) {
compBuilder.setSharedWrite();
} else {
compBuilder.setExclWrite();
}
compBuilder.setOperationType(DataOperationType.valueOf(output.getWriteType().name()));
break;
case DDL_NO_LOCK:
// No lock required here
continue;
default:
throw new RuntimeException("Unknown write type " + output.getWriteType().toString());
}
if (t != null) {
compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t));
}
compBuilder.setIsDynamicPartitionWrite(output.isDynamicPartitionWrite());
LockComponent comp = compBuilder.build();
LOG.debug("Adding lock component to lock request " + comp.toString());
lockComponents.add(comp);
}
return lockComponents;
}
Aggregations