use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.
the class TestDbTxnManager2 method testFairness.
@Test
public void testFairness() throws Exception {
dropTable(new String[] { "T6" });
CommandProcessorResponse cpr = driver.run("create table if not exists T6(a int)");
checkCmdOnDriver(cpr);
cpr = driver.compileAndRespond("select a from T6");
checkCmdOnDriver(cpr);
// gets S lock on T6
txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
swapTxnManager(txnMgr2);
cpr = driver.compileAndRespond("drop table if exists T6");
checkCmdOnDriver(cpr);
// tries to get X lock on T6 and gets Waiting state
LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
List<ShowLocksResponseElement> locks = getLocks();
Assert.assertEquals("Unexpected lock count", 2, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
swapTxnManager(txnMgr3);
// this should block behind the X lock on T6
// this is a contrived example, in practice this query would of course fail after drop table
cpr = driver.compileAndRespond("select a from T6");
checkCmdOnDriver(cpr);
// gets S lock on T6
((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);
locks = getLocks();
Assert.assertEquals("Unexpected lock count", 3, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T6", null, locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
}
use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.
the class TestDbTxnManager2 method basicBlocking.
@Test
public void basicBlocking() throws Exception {
dropTable(new String[] { "T6" });
CommandProcessorResponse cpr = driver.run("create table if not exists T6(a int)");
checkCmdOnDriver(cpr);
cpr = driver.compileAndRespond("select a from T6");
checkCmdOnDriver(cpr);
// gets S lock on T6
txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
List<HiveLock> selectLocks = ctx.getHiveLocks();
HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
swapTxnManager(txnMgr2);
cpr = driver.compileAndRespond("drop table if exists T6");
checkCmdOnDriver(cpr);
// tries to get X lock on T1 and gets Waiting state
LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
List<ShowLocksResponseElement> locks = getLocks();
Assert.assertEquals("Unexpected lock count", 2, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
// release S on T6
txnMgr.rollbackTxn();
// attempt to X on T6 again - succeed
lockState = ((DbLockManager) txnMgr.getLockManager()).checkLock(locks.get(1).getLockid());
locks = getLocks();
Assert.assertEquals("Unexpected lock count", 1, locks.size());
checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T6", null, locks);
txnMgr2.rollbackTxn();
cpr = driver.run("drop table if exists T6");
locks = getLocks();
Assert.assertEquals("Unexpected number of locks found", 0, locks.size());
checkCmdOnDriver(cpr);
}
use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.
the class TestDbTxnManager2 method testFairness2.
/**
* T7 is a table with 2 partitions
* 1. run select from T7
* 2. run drop partition from T7
* concurrently with 1 starting first so that 2 blocks
* 3. start another concurrent select on T7 - it should block behind waiting X (from drop) - LM should be fair
* 4. finish #1 so that drop unblocks
* 5. rollback the drop to release its X lock
* 6. # should unblock
*/
@Test
public void testFairness2() throws Exception {
dropTable(new String[] { "T7" });
CommandProcessorResponse cpr = driver.run("create table if not exists T7 (a int) " + "partitioned by (p int) stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
checkCmdOnDriver(driver.run(// create 2 partitions
"insert into T7 partition(p) values(1,1),(1,2)"));
cpr = driver.compileAndRespond("select a from T7 ");
checkCmdOnDriver(cpr);
// gets S lock on T7
txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
swapTxnManager(txnMgr2);
cpr = driver.compileAndRespond("alter table T7 drop partition (p=1)");
checkCmdOnDriver(cpr);
// tries to get X lock on T7.p=1 and gets Waiting state
LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
List<ShowLocksResponseElement> locks = getLocks();
Assert.assertEquals("Unexpected lock count", 4, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T7", "p=1", locks);
HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
swapTxnManager(txnMgr3);
// this should block behind the X lock on T7.p=1
cpr = driver.compileAndRespond("select a from T7");
checkCmdOnDriver(cpr);
// tries to get S lock on T7, S on T7.p=1 and S on T7.p=2
((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);
locks = getLocks();
Assert.assertEquals("Unexpected lock count", 7, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", null, locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=2", locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T7", "p=1", locks);
// release locks from "select a from T7" - to unblock hte drop partition
txnMgr.commitTxn();
// retest the the "drop partiton" X lock
lockState = ((DbLockManager) txnMgr2.getLockManager()).checkLock(locks.get(6).getLockid());
locks = getLocks();
Assert.assertEquals("Unexpected lock count", 4, locks.size());
checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", null, locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=2", locks);
// release the X lock on T7.p=1
txnMgr2.rollbackTxn();
// re-test the locks
// S lock on T7
lockState = ((DbLockManager) txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid());
locks = getLocks();
Assert.assertEquals("Unexpected lock count", 3, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
}
use of org.apache.hadoop.hive.metastore.api.LockState in project presto by prestodb.
the class ThriftHiveMetastore method lock.
@Override
public long lock(MetastoreContext metastoreContext, String databaseName, String tableName) {
try {
final LockComponent lockComponent = new LockComponent(EXCLUSIVE, LockLevel.TABLE, databaseName);
lockComponent.setTablename(tableName);
final LockRequest lockRequest = new LockRequest(Lists.newArrayList(lockComponent), metastoreContext.getUsername(), InetAddress.getLocalHost().getHostName());
LockResponse lockResponse = stats.getLock().wrap(() -> getMetastoreClientThenCall(metastoreContext, client -> client.lock(lockRequest))).call();
LockState state = lockResponse.getState();
long lockId = lockResponse.getLockid();
final AtomicBoolean acquired = new AtomicBoolean(state.equals(ACQUIRED));
try {
if (state.equals(WAITING)) {
retry().maxAttempts(Integer.MAX_VALUE - 100).stopOnIllegalExceptions().exceptionMapper(e -> {
if (e instanceof WaitingForLockException) {
// only retry on waiting for lock exception
return e;
} else {
return new IllegalStateException(e.getMessage(), e);
}
}).run("lock", stats.getLock().wrap(() -> getMetastoreClientThenCall(metastoreContext, client -> {
LockResponse response = client.checkLock(new CheckLockRequest(lockId));
LockState newState = response.getState();
if (newState.equals(WAITING)) {
throw new WaitingForLockException("Waiting for lock.");
} else if (newState.equals(ACQUIRED)) {
acquired.set(true);
} else {
throw new RuntimeException(String.format("Failed to acquire lock: %s", newState.name()));
}
return null;
})));
}
} finally {
if (!acquired.get()) {
unlock(metastoreContext, lockId);
}
}
if (!acquired.get()) {
throw new RuntimeException("Failed to acquire lock");
}
return lockId;
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
} catch (Exception e) {
throw propagate(e);
}
}
use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.
the class HiveTableOperations method acquireLock.
@VisibleForTesting
long acquireLock() throws UnknownHostException, TException, InterruptedException {
final LockComponent lockComponent = new LockComponent(LockType.EXCL_WRITE, LockLevel.TABLE, database);
lockComponent.setTablename(tableName);
final LockRequest lockRequest = new LockRequest(Lists.newArrayList(lockComponent), System.getProperty("user.name"), InetAddress.getLocalHost().getHostName());
LockResponse lockResponse = metaClients.run(client -> client.lock(lockRequest));
AtomicReference<LockState> state = new AtomicReference<>(lockResponse.getState());
long lockId = lockResponse.getLockid();
final long start = System.currentTimeMillis();
long duration = 0;
boolean timeout = false;
try {
if (state.get().equals(LockState.WAITING)) {
// Retry count is the typical "upper bound of retries" for Tasks.run() function. In fact, the maximum number of
// attempts the Tasks.run() would try is `retries + 1`. Here, for checking locks, we use timeout as the
// upper bound of retries. So it is just reasonable to set a large retry count. However, if we set
// Integer.MAX_VALUE, the above logic of `retries + 1` would overflow into Integer.MIN_VALUE. Hence,
// the retry is set conservatively as `Integer.MAX_VALUE - 100` so it doesn't hit any boundary issues.
Tasks.foreach(lockId).retry(Integer.MAX_VALUE - 100).exponentialBackoff(lockCheckMinWaitTime, lockCheckMaxWaitTime, lockAcquireTimeout, 1.5).throwFailureWhenFinished().onlyRetryOn(WaitingForLockException.class).run(id -> {
try {
LockResponse response = metaClients.run(client -> client.checkLock(id));
LockState newState = response.getState();
state.set(newState);
if (newState.equals(LockState.WAITING)) {
throw new WaitingForLockException("Waiting for lock.");
}
} catch (InterruptedException e) {
// Clear the interrupt status flag
Thread.interrupted();
LOG.warn("Interrupted while waiting for lock.", e);
}
}, TException.class);
}
} catch (WaitingForLockException waitingForLockException) {
timeout = true;
duration = System.currentTimeMillis() - start;
} finally {
if (!state.get().equals(LockState.ACQUIRED)) {
unlock(Optional.of(lockId));
}
}
// timeout and do not have lock acquired
if (timeout && !state.get().equals(LockState.ACQUIRED)) {
throw new CommitFailedException("Timed out after %s ms waiting for lock on %s.%s", duration, database, tableName);
}
if (!state.get().equals(LockState.ACQUIRED)) {
throw new CommitFailedException("Could not acquire the lock on %s.%s, " + "lock request ended in state %s", database, tableName, state);
}
return lockId;
}
Aggregations