Examples with LockState org.apache.hadoop.hive.metastore.api.LockState used on opensource projects

Example 6 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class AlterMaterializedViewRebuildAnalyzer method getRewrittenAST.

private ASTNode getRewrittenAST(TableName tableName) throws SemanticException {
    ASTNode rewrittenAST;
    // We need to go lookup the table and get the select statement and then parse it.
    try {
        Table table = getTableObjectByName(tableName.getNotEmptyDbTable(), true);
        if (!table.isMaterializedView()) {
            // Cannot rebuild not materialized view
            throw new SemanticException(ErrorMsg.REBUILD_NO_MATERIALIZED_VIEW);
        }
        // We need to use the expanded text for the materialized view, as it will contain
        // the qualified table aliases, etc.
        String viewText = table.getViewExpandedText();
        if (viewText.trim().isEmpty()) {
            throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY);
        }
        Context ctx = new Context(queryState.getConf());
        String rewrittenInsertStatement = String.format(REWRITTEN_INSERT_STATEMENT, tableName.getEscapedNotEmptyDbTable(), viewText);
        rewrittenAST = ParseUtils.parse(rewrittenInsertStatement, ctx);
        this.ctx.addSubContext(ctx);
        if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(table)) {
            // Acquire lock for the given materialized view. Only one rebuild per materialized view can be triggered at a
            // given time, as otherwise we might produce incorrect results if incremental maintenance is triggered.
            HiveTxnManager txnManager = getTxnMgr();
            LockState state;
            try {
                state = txnManager.acquireMaterializationRebuildLock(tableName.getDb(), tableName.getTable(), txnManager.getCurrentTxnId()).getState();
            } catch (LockException e) {
                throw new SemanticException("Exception acquiring lock for rebuilding the materialized view", e);
            }
            if (state != LockState.ACQUIRED) {
                throw new SemanticException("Another process is rebuilding the materialized view " + tableName.getNotEmptyDbTable());
            }
        }
    } catch (Exception e) {
        throw new SemanticException(e);
    }
    return rewrittenAST;
}

Example 7 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class Lock method internalAcquire.

private long internalAcquire(Long transactionId) throws LockException {
    int attempts = 0;
    LockRequest request = buildLockRequest(transactionId);
    do {
        LockResponse response = null;
        try {
            response = metaStoreClient.lock(request);
        } catch (TException e) {
            throw new LockException("Unable to acquire lock for tables: [" + join(tables) + "]", e);
        }
        if (response != null) {
            LockState state = response.getState();
            if (state == LockState.NOT_ACQUIRED || state == LockState.ABORT) {
                // I expect we'll only see NOT_ACQUIRED here?
                break;
            }
            if (state == LockState.ACQUIRED) {
                LOG.debug("Acquired lock {}", response.getLockid());
                return response.getLockid();
            }
            if (state == LockState.WAITING) {
                try {
                    Thread.sleep(TimeUnit.SECONDS.toMillis(retryWaitSeconds));
                } catch (InterruptedException e) {
                }
            }
        }
        attempts++;
    } while (attempts < lockRetries);
    throw new LockException("Could not acquire lock on tables: [" + join(tables) + "]");
}

Example 8 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class TestDbTxnManager2 method checkExpectedLocks.

/**
 * collection of queries where we ensure that we get the locks that are expected
 * @throws Exception
 */
@Test
public void checkExpectedLocks() throws Exception {
    dropTable(new String[] { "acidPart", "nonAcidPart" });
    CommandProcessorResponse cpr = null;
    cpr = driver.run("create table acidPart(a int, b int) partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='false')");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("insert into nonAcidPart partition(p) values(1,2,3)");
    checkCmdOnDriver(cpr);
    LockState lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
    txnMgr.rollbackTxn();
    ;
    cpr = driver.compileAndRespond("insert into nonAcidPart partition(p=1) values(5,6)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
    txnMgr.rollbackTxn();
    cpr = driver.compileAndRespond("insert into acidPart partition(p) values(1,2,3)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
    txnMgr.rollbackTxn();
    cpr = driver.compileAndRespond("insert into acidPart partition(p=1) values(5,6)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
    txnMgr.rollbackTxn();
    cpr = driver.compileAndRespond("update acidPart set b = 17 where a = 1");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "acidPart", null, locks);
    txnMgr.rollbackTxn();
    cpr = driver.compileAndRespond("update acidPart set b = 17 where p = 1");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    // https://issues.apache.org/jira/browse/HIVE-13212
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "acidPart", null, locks);
    txnMgr.rollbackTxn();
}

Example 9 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class TestDbTxnManager2 method checkExpectedLocks2.

/**
 * Check to make sure we acquire proper locks for queries involving acid and non-acid tables
 */
@Test
public void checkExpectedLocks2() throws Exception {
    dropTable(new String[] { "tab_acid", "tab_not_acid" });
    checkCmdOnDriver(driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')"));
    checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " + "clustered by (na) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='false')"));
    checkCmdOnDriver(driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')"));
    checkCmdOnDriver(driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')"));
    txnMgr.openTxn(ctx, "T1");
    checkCmdOnDriver(driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 6, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "T2");
    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)"));
    LockState ls = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false);
    locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 8, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "_dummy_database", "_dummy_table", null, locks);
    // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources.
    // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID semantics
    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, false);
    HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr3.openTxn(ctx, "T3");
    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)"));
    ((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false);
    locks = getLocks(txnMgr3);
    Assert.assertEquals("Unexpected lock count", 10, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.SHARED_READ, LockState.WAITING, "_dummy_database", "_dummy_table", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks);
    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
}

Example 10 with LockState

use of org.apache.hadoop.hive.metastore.api.LockState in project hive by apache.

the class DbTxnManager method acquireLocks.

/**
 * Normally client should call {@link #acquireLocks(org.apache.hadoop.hive.ql.QueryPlan, org.apache.hadoop.hive.ql.Context, String)}
 * @param isBlocking if false, the method will return immediately; thus the locks may be in LockState.WAITING
 * @return null if no locks were needed
 */
@VisibleForTesting
LockState acquireLocks(QueryPlan plan, Context ctx, String username, boolean isBlocking) throws LockException {
    init();
    // Make sure we've built the lock manager
    getLockManager();
    verifyState(plan);
    queryId = plan.getQueryId();
    switch(plan.getOperation()) {
        case SET_AUTOCOMMIT:
            /**
             *This is here for documentation purposes.  This TM doesn't support this - only has one
             * mode of operation documented at {@link DbTxnManager#isExplicitTransaction}
             */
            return null;
    }
    LockRequestBuilder rqstBuilder = new LockRequestBuilder(queryId);
    // link queryId to txnId
    LOG.info("Setting lock request transaction to " + JavaUtils.txnIdToString(txnId) + " for queryId=" + queryId);
    rqstBuilder.setTransactionId(txnId).setUser(username);
    rqstBuilder.setZeroWaitReadEnabled(!conf.getBoolVar(HiveConf.ConfVars.TXN_OVERWRITE_X_LOCK) || !conf.getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK));
    // this operation.
    if (plan.getInputs().isEmpty() && plan.getOutputs().isEmpty()) {
        LOG.debug("No locks needed for queryId=" + queryId);
        return null;
    }
    List<LockComponent> lockComponents = AcidUtils.makeLockComponents(plan.getOutputs(), plan.getInputs(), ctx.getOperation(), conf);
    lockComponents.addAll(getGlobalLocks(ctx.getConf()));
    // It's possible there's nothing to lock even if we have w/r entities.
    if (lockComponents.isEmpty()) {
        LOG.debug("No locks needed for queryId=" + queryId);
        return null;
    }
    rqstBuilder.addLockComponents(lockComponents);
    List<HiveLock> locks = new ArrayList<HiveLock>(1);
    LockState lockState = lockMgr.lock(rqstBuilder.build(), queryId, isBlocking, locks);
    ctx.setHiveLocks(locks);
    return lockState;
}