Search in sources :

Example 1 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class TestDbTxnManager2 method testFairness.

@Test
public void testFairness() throws Exception {
    dropTable(new String[] { "T6" });
    CommandProcessorResponse cpr = driver.run("create table if not exists T6(a int)");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("select a from T6");
    checkCmdOnDriver(cpr);
    // gets S lock on T6
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr2);
    cpr = driver.compileAndRespond("drop table if exists T6");
    checkCmdOnDriver(cpr);
    // tries to get X lock on T6 and gets Waiting state
    LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
    HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr3);
    // this should block behind the X lock on  T6
    // this is a contrived example, in practice this query would of course fail after drop table
    cpr = driver.compileAndRespond("select a from T6");
    checkCmdOnDriver(cpr);
    // gets S lock on T6
    ((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 3, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T6", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
}
Also used : CommandProcessorResponse(org.apache.hadoop.hive.ql.processors.CommandProcessorResponse) LockState(org.apache.hadoop.hive.metastore.api.LockState) ShowLocksResponseElement(org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement) Test(org.junit.Test)

Example 2 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class TestDbTxnManager2 method basicBlocking.

@Test
public void basicBlocking() throws Exception {
    dropTable(new String[] { "T6" });
    CommandProcessorResponse cpr = driver.run("create table if not exists T6(a int)");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("select a from T6");
    checkCmdOnDriver(cpr);
    // gets S lock on T6
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    List<HiveLock> selectLocks = ctx.getHiveLocks();
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr2);
    cpr = driver.compileAndRespond("drop table if exists T6");
    checkCmdOnDriver(cpr);
    // tries to get X lock on T1 and gets Waiting state
    LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T6", null, locks);
    // release S on T6
    txnMgr.rollbackTxn();
    // attempt to X on T6 again - succeed
    lockState = ((DbLockManager) txnMgr.getLockManager()).checkLock(locks.get(1).getLockid());
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T6", null, locks);
    txnMgr2.rollbackTxn();
    cpr = driver.run("drop table if exists T6");
    locks = getLocks();
    Assert.assertEquals("Unexpected number of locks found", 0, locks.size());
    checkCmdOnDriver(cpr);
}
Also used : CommandProcessorResponse(org.apache.hadoop.hive.ql.processors.CommandProcessorResponse) LockState(org.apache.hadoop.hive.metastore.api.LockState) ShowLocksResponseElement(org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement) Test(org.junit.Test)

Example 3 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class TestDbTxnManager2 method testFairness2.

/**
 * T7 is a table with 2 partitions
 * 1. run select from T7
 * 2. run drop partition from T7
 * concurrently with 1 starting first so that 2 blocks
 * 3. start another concurrent select on T7 - it should block behind waiting X (from drop) - LM should be fair
 * 4. finish #1 so that drop unblocks
 * 5. rollback the drop to release its X lock
 * 6. # should unblock
 */
@Test
public void testFairness2() throws Exception {
    dropTable(new String[] { "T7" });
    CommandProcessorResponse cpr = driver.run("create table if not exists T7 (a int) " + "partitioned by (p int) stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    checkCmdOnDriver(driver.run(// create 2 partitions
    "insert into T7 partition(p) values(1,1),(1,2)"));
    cpr = driver.compileAndRespond("select a from T7 ");
    checkCmdOnDriver(cpr);
    // gets S lock on T7
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr2);
    cpr = driver.compileAndRespond("alter table T7 drop partition (p=1)");
    checkCmdOnDriver(cpr);
    // tries to get X lock on T7.p=1 and gets Waiting state
    LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 4, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T7", "p=1", locks);
    HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr3);
    // this should block behind the X lock on  T7.p=1
    cpr = driver.compileAndRespond("select a from T7");
    checkCmdOnDriver(cpr);
    // tries to get S lock on T7, S on T7.p=1 and S on T7.p=2
    ((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 7, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", null, locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=2", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T7", "p=1", locks);
    // release locks from "select a from T7" - to unblock hte drop partition
    txnMgr.commitTxn();
    // retest the the "drop partiton" X lock
    lockState = ((DbLockManager) txnMgr2.getLockManager()).checkLock(locks.get(6).getLockid());
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 4, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", null, locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "default", "T7", "p=2", locks);
    // release the X lock on T7.p=1
    txnMgr2.rollbackTxn();
    // re-test the locks
    // S lock on T7
    lockState = ((DbLockManager) txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid());
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 3, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T7", "p=2", locks);
}
Also used : CommandProcessorResponse(org.apache.hadoop.hive.ql.processors.CommandProcessorResponse) LockState(org.apache.hadoop.hive.metastore.api.LockState) ShowLocksResponseElement(org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement) Test(org.junit.Test)

Example 4 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project presto by prestodb.

the class ThriftHiveMetastore method lock.

@Override
public long lock(MetastoreContext metastoreContext, String databaseName, String tableName) {
    try {
        final LockComponent lockComponent = new LockComponent(EXCLUSIVE, LockLevel.TABLE, databaseName);
        lockComponent.setTablename(tableName);
        final LockRequest lockRequest = new LockRequest(Lists.newArrayList(lockComponent), metastoreContext.getUsername(), InetAddress.getLocalHost().getHostName());
        LockResponse lockResponse = stats.getLock().wrap(() -> getMetastoreClientThenCall(metastoreContext, client -> client.lock(lockRequest))).call();
        LockState state = lockResponse.getState();
        long lockId = lockResponse.getLockid();
        final AtomicBoolean acquired = new AtomicBoolean(state.equals(ACQUIRED));
        try {
            if (state.equals(WAITING)) {
                retry().maxAttempts(Integer.MAX_VALUE - 100).stopOnIllegalExceptions().exceptionMapper(e -> {
                    if (e instanceof WaitingForLockException) {
                        // only retry on waiting for lock exception
                        return e;
                    } else {
                        return new IllegalStateException(e.getMessage(), e);
                    }
                }).run("lock", stats.getLock().wrap(() -> getMetastoreClientThenCall(metastoreContext, client -> {
                    LockResponse response = client.checkLock(new CheckLockRequest(lockId));
                    LockState newState = response.getState();
                    if (newState.equals(WAITING)) {
                        throw new WaitingForLockException("Waiting for lock.");
                    } else if (newState.equals(ACQUIRED)) {
                        acquired.set(true);
                    } else {
                        throw new RuntimeException(String.format("Failed to acquire lock: %s", newState.name()));
                    }
                    return null;
                })));
            }
        } finally {
            if (!acquired.get()) {
                unlock(metastoreContext, lockId);
            }
        }
        if (!acquired.get()) {
            throw new RuntimeException("Failed to acquire lock");
        }
        return lockId;
    } catch (TException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    } catch (Exception e) {
        throw propagate(e);
    }
}
Also used : SchemaAlreadyExistsException(com.facebook.presto.hive.SchemaAlreadyExistsException) EXCLUSIVE(org.apache.hadoop.hive.metastore.api.LockType.EXCLUSIVE) NUMBER_OF_NON_NULL_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_NON_NULL_VALUES) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) Throwables.throwIfUnchecked(com.google.common.base.Throwables.throwIfUnchecked) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) InetAddress(java.net.InetAddress) MetastoreUtil.getHiveBasicStatistics(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics) Sets.difference(com.google.common.collect.Sets.difference) MAX_VALUE_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE_SIZE_IN_BYTES) Map(java.util.Map) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) ThriftMetastoreUtil.toMetastoreApiPartition(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiPartition) HiveBasicStatistics(com.facebook.presto.hive.HiveBasicStatistics) TableAlreadyExistsException(com.facebook.presto.hive.TableAlreadyExistsException) InvalidInputException(org.apache.hadoop.hive.metastore.api.InvalidInputException) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) ThreadSafe(javax.annotation.concurrent.ThreadSafe) TypeUtils.isNumericType(com.facebook.presto.common.type.TypeUtils.isNumericType) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) MIN_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MIN_VALUE) NUMBER_OF_DISTINCT_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_DISTINCT_VALUES) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) Iterables(com.google.common.collect.Iterables) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) Flatten(org.weakref.jmx.Flatten) ACQUIRED(org.apache.hadoop.hive.metastore.api.LockState.ACQUIRED) HIVE_FILTER_FIELD_PARAMS(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS) Callable(java.util.concurrent.Callable) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) MetastoreUtil.convertPredicateToParts(com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) OptionalLong(java.util.OptionalLong) Lists(com.google.common.collect.Lists) Managed(org.weakref.jmx.Managed) LockState(org.apache.hadoop.hive.metastore.api.LockState) UnlockRequest(org.apache.hadoop.hive.metastore.api.UnlockRequest) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) CheckLockRequest(org.apache.hadoop.hive.metastore.api.CheckLockRequest) RetryDriver(com.facebook.presto.hive.RetryDriver) PrivilegeGrantInfo(org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo) UnknownDBException(org.apache.hadoop.hive.metastore.api.UnknownDBException) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) TException(org.apache.thrift.TException) PrincipalType(org.apache.hadoop.hive.metastore.api.PrincipalType) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Domain(com.facebook.presto.common.predicate.Domain) Table(org.apache.hadoop.hive.metastore.api.Table) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) TableType(org.apache.hadoop.hive.metastore.TableType) HiveObjectRef(org.apache.hadoop.hive.metastore.api.HiveObjectRef) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) RowType(com.facebook.presto.common.type.RowType) NUMBER_OF_TRUE_VALUES(com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_TRUE_VALUES) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) PartitionNotFoundException(com.facebook.presto.hive.PartitionNotFoundException) LockRequest(org.apache.hadoop.hive.metastore.api.LockRequest) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) ThriftMetastoreUtil.fromMetastoreApiPrincipalType(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiPrincipalType) SchemaNotFoundException(com.facebook.presto.spi.SchemaNotFoundException) LockComponent(org.apache.hadoop.hive.metastore.api.LockComponent) PRESTO_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG) HiveViewNotSupportedException(com.facebook.presto.hive.HiveViewNotSupportedException) Collectors.toSet(java.util.stream.Collectors.toSet) PrivilegeBag(org.apache.hadoop.hive.metastore.api.PrivilegeBag) ImmutableSet(com.google.common.collect.ImmutableSet) WAITING(org.apache.hadoop.hive.metastore.api.LockState.WAITING) ImmutableMap(com.google.common.collect.ImmutableMap) ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) LockResponse(org.apache.hadoop.hive.metastore.api.LockResponse) MAX_VALUE(com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE) Collectors(java.util.stream.Collectors) ThriftMetastoreUtil.createMetastoreColumnStatistics(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.createMetastoreColumnStatistics) String.format(java.lang.String.format) List(java.util.List) RoleGrant(com.facebook.presto.spi.security.RoleGrant) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) ThriftMetastoreUtil.parsePrivilege(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.parsePrivilege) HiveObjectPrivilege(org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege) TOTAL_SIZE_IN_BYTES(com.facebook.presto.spi.statistics.ColumnStatisticType.TOTAL_SIZE_IN_BYTES) ThriftMetastoreUtil.fromMetastoreApiTable(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable) MapType(com.facebook.presto.common.type.MapType) Column(com.facebook.presto.hive.metastore.Column) HiveType(com.facebook.presto.hive.HiveType) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PrestoException(com.facebook.presto.spi.PrestoException) Partition(org.apache.hadoop.hive.metastore.api.Partition) Function(java.util.function.Function) OWNERSHIP(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP) Inject(javax.inject.Inject) HashSet(java.util.HashSet) LockLevel(org.apache.hadoop.hive.metastore.api.LockLevel) ImmutableList(com.google.common.collect.ImmutableList) ALREADY_EXISTS(com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS) Objects.requireNonNull(java.util.Objects.requireNonNull) MetastoreUtil.updateStatisticsParameters(com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters) MetastoreClientConfig(com.facebook.presto.hive.MetastoreClientConfig) Type(com.facebook.presto.common.type.Type) ThriftMetastoreUtil.fromRolePrincipalGrants(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromRolePrincipalGrants) USER(com.facebook.presto.spi.security.PrincipalType.USER) TABLE(org.apache.hadoop.hive.metastore.api.HiveObjectType.TABLE) Iterator(java.util.Iterator) ThriftMetastoreUtil.fromPrestoPrincipalType(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromPrestoPrincipalType) UnknownTableException(org.apache.hadoop.hive.metastore.api.UnknownTableException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Database(org.apache.hadoop.hive.metastore.api.Database) TException(org.apache.thrift.TException) LockComponent(org.apache.hadoop.hive.metastore.api.LockComponent) CheckLockRequest(org.apache.hadoop.hive.metastore.api.CheckLockRequest) PrestoException(com.facebook.presto.spi.PrestoException) SchemaAlreadyExistsException(com.facebook.presto.hive.SchemaAlreadyExistsException) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) TableAlreadyExistsException(com.facebook.presto.hive.TableAlreadyExistsException) InvalidInputException(org.apache.hadoop.hive.metastore.api.InvalidInputException) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) UnknownDBException(org.apache.hadoop.hive.metastore.api.UnknownDBException) TException(org.apache.thrift.TException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) PartitionNotFoundException(com.facebook.presto.hive.PartitionNotFoundException) SchemaNotFoundException(com.facebook.presto.spi.SchemaNotFoundException) HiveViewNotSupportedException(com.facebook.presto.hive.HiveViewNotSupportedException) PrestoException(com.facebook.presto.spi.PrestoException) UnknownTableException(org.apache.hadoop.hive.metastore.api.UnknownTableException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LockResponse(org.apache.hadoop.hive.metastore.api.LockResponse) LockState(org.apache.hadoop.hive.metastore.api.LockState) CheckLockRequest(org.apache.hadoop.hive.metastore.api.CheckLockRequest) LockRequest(org.apache.hadoop.hive.metastore.api.LockRequest)

Example 5 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class HiveTableOperations method acquireLock.

@VisibleForTesting
long acquireLock() throws UnknownHostException, TException, InterruptedException {
    final LockComponent lockComponent = new LockComponent(LockType.EXCL_WRITE, LockLevel.TABLE, database);
    lockComponent.setTablename(tableName);
    final LockRequest lockRequest = new LockRequest(Lists.newArrayList(lockComponent), System.getProperty("user.name"), InetAddress.getLocalHost().getHostName());
    LockResponse lockResponse = metaClients.run(client -> client.lock(lockRequest));
    AtomicReference<LockState> state = new AtomicReference<>(lockResponse.getState());
    long lockId = lockResponse.getLockid();
    final long start = System.currentTimeMillis();
    long duration = 0;
    boolean timeout = false;
    try {
        if (state.get().equals(LockState.WAITING)) {
            // Retry count is the typical "upper bound of retries" for Tasks.run() function. In fact, the maximum number of
            // attempts the Tasks.run() would try is `retries + 1`. Here, for checking locks, we use timeout as the
            // upper bound of retries. So it is just reasonable to set a large retry count. However, if we set
            // Integer.MAX_VALUE, the above logic of `retries + 1` would overflow into Integer.MIN_VALUE. Hence,
            // the retry is set conservatively as `Integer.MAX_VALUE - 100` so it doesn't hit any boundary issues.
            Tasks.foreach(lockId).retry(Integer.MAX_VALUE - 100).exponentialBackoff(lockCheckMinWaitTime, lockCheckMaxWaitTime, lockAcquireTimeout, 1.5).throwFailureWhenFinished().onlyRetryOn(WaitingForLockException.class).run(id -> {
                try {
                    LockResponse response = metaClients.run(client -> client.checkLock(id));
                    LockState newState = response.getState();
                    state.set(newState);
                    if (newState.equals(LockState.WAITING)) {
                        throw new WaitingForLockException("Waiting for lock.");
                    }
                } catch (InterruptedException e) {
                    // Clear the interrupt status flag
                    Thread.interrupted();
                    LOG.warn("Interrupted while waiting for lock.", e);
                }
            }, TException.class);
        }
    } catch (WaitingForLockException waitingForLockException) {
        timeout = true;
        duration = System.currentTimeMillis() - start;
    } finally {
        if (!state.get().equals(LockState.ACQUIRED)) {
            unlock(Optional.of(lockId));
        }
    }
    // timeout and do not have lock acquired
    if (timeout && !state.get().equals(LockState.ACQUIRED)) {
        throw new CommitFailedException("Timed out after %s ms waiting for lock on %s.%s", duration, database, tableName);
    }
    if (!state.get().equals(LockState.ACQUIRED)) {
        throw new CommitFailedException("Could not acquire the lock on %s.%s, " + "lock request ended in state %s", database, tableName, state);
    }
    return lockId;
}
Also used : LockComponent(org.apache.hadoop.hive.metastore.api.LockComponent) LockResponse(org.apache.hadoop.hive.metastore.api.LockResponse) AtomicReference(java.util.concurrent.atomic.AtomicReference) LockState(org.apache.hadoop.hive.metastore.api.LockState) LockRequest(org.apache.hadoop.hive.metastore.api.LockRequest) CommitFailedException(org.apache.iceberg.exceptions.CommitFailedException) VisibleForTesting(org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting)

Aggregations

LockState (org.apache.hadoop.hive.metastore.api.LockState)10 ShowLocksResponseElement (org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement)5 Test (org.junit.Test)5 CommandProcessorResponse (org.apache.hadoop.hive.ql.processors.CommandProcessorResponse)4 LockComponent (org.apache.hadoop.hive.metastore.api.LockComponent)3 LockRequest (org.apache.hadoop.hive.metastore.api.LockRequest)3 LockResponse (org.apache.hadoop.hive.metastore.api.LockResponse)3 Domain (com.facebook.presto.common.predicate.Domain)1 ArrayType (com.facebook.presto.common.type.ArrayType)1 BOOLEAN (com.facebook.presto.common.type.BooleanType.BOOLEAN)1 Chars.isCharType (com.facebook.presto.common.type.Chars.isCharType)1 DATE (com.facebook.presto.common.type.DateType.DATE)1 MapType (com.facebook.presto.common.type.MapType)1 RowType (com.facebook.presto.common.type.RowType)1 TIMESTAMP (com.facebook.presto.common.type.TimestampType.TIMESTAMP)1 Type (com.facebook.presto.common.type.Type)1 TypeUtils.isNumericType (com.facebook.presto.common.type.TypeUtils.isNumericType)1 VARBINARY (com.facebook.presto.common.type.VarbinaryType.VARBINARY)1 Varchars.isVarcharType (com.facebook.presto.common.type.Varchars.isVarcharType)1 HiveBasicStatistics (com.facebook.presto.hive.HiveBasicStatistics)1