Examples with ShowLocksResponseElement org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement used on opensource projects

Example 26 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testLockBlockedBy.

/**
   * check that locks in Waiting state show what they are waiting on
   * This test is somewhat abusive in that it make DbLockManager retain locks for 2
   * different queries (which are not part of the same transaction) which can never
   * happen in real use cases... but it makes testing convenient.
   * @throws Exception
   */
@Test
public void testLockBlockedBy() throws Exception {
    dropTable(new String[] { "TAB_BLOCKED" });
    CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("select * from TAB_BLOCKED");
    checkCmdOnDriver(cpr);
    txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks);
    cpr = driver.compileAndRespond("drop table TAB_BLOCKED");
    checkCmdOnDriver(cpr);
    //make non-blocking
    ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks);
    Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId());
    Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId());
}

Example 27 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method checkExpectedLocks2.

/**
   * Check to make sure we acquire proper locks for queries involving acid and non-acid tables
   */
@Test
public void checkExpectedLocks2() throws Exception {
    dropTable(new String[] { "tab_acid", "tab_not_acid" });
    checkCmdOnDriver(driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')"));
    checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " + "clustered by (na) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='false')"));
    checkCmdOnDriver(driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')"));
    checkCmdOnDriver(driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')"));
    txnMgr.openTxn(ctx, "T1");
    checkCmdOnDriver(driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 6, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "T2");
    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)"));
    LockState ls = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false);
    locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 7, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
    // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources.
    // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID semantics
    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, false);
    HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr3.openTxn(ctx, "T3");
    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)"));
    ((DbTxnManager) txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false);
    locks = getLocks(txnMgr3);
    Assert.assertEquals("Unexpected lock count", 8, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
}

Example 28 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method lockConflictDbTable.

@Test
public void lockConflictDbTable() throws Exception {
    dropTable(new String[] { "temp.T7" });
    CommandProcessorResponse cpr = driver.run("create database if not exists temp");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table if not exists temp.T7(a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("update temp.T7 set a = 5 where b = 6");
    checkCmdOnDriver(cpr);
    txnMgr.openTxn(ctx, "Fifer");
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    checkCmdOnDriver(driver.compileAndRespond("drop database if exists temp"));
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    //txnMgr2.openTxn("Fiddler");
    //gets SS lock on T7
    ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "temp", "T7", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "temp", null, null, locks);
    txnMgr.commitTxn();
    ((DbLockManager) txnMgr.getLockManager()).checkLock(locks.get(1).getLockid());
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "temp", null, null, locks);
    List<HiveLock> xLock = new ArrayList<HiveLock>(0);
    xLock.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr2.getLockManager().releaseLocks(xLock);
}

Example 29 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking4.

/**
   * txns overlap, update same resource, simulate multi-stmt txn case
   * Also tests that we kill txn when it tries to acquire lock if we already know it will not be committed
   */
@Test
public void testWriteSetTracking4() throws Exception {
    dropTable(new String[] { "TAB_PART", "TAB2" });
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    txnMgr.openTxn(ctx, "Long Running");
    checkCmdOnDriver(driver.compileAndRespond("select a from  TAB_PART where p = 'blah'"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    //for some reason this just locks the table; if I alter table to add this partition, then 
    //we end up locking both table and partition with share_read.  (Plan has 2 ReadEntities)...?
    //same for other locks below
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "Short Running");
    //no such partition
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'blah'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "Short Running");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    //Short Running updated nothing, so we expect 0 rows in WRITE_SET
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    txnMgr2.openTxn(ctx, "T3");
    //pretend this partition exists
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T3");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    //since TAB2 is empty
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab2", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    //simulate partition update
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    AcidWriteSetService houseKeeper = new AcidWriteSetService();
    TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
    //since T3 overlaps with Long Running (still open) GC does nothing
    Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    //no rows match
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    //so generate empty Dyn Part call
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr.commitTxn();
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 0, locks.size());
    TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
}

Example 30 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class DDLTask method dumpLockInfo.

public static void dumpLockInfo(DataOutputStream os, ShowLocksResponse rsp) throws IOException {
    // Write a header
    os.writeBytes("Lock ID");
    os.write(separator);
    os.writeBytes("Database");
    os.write(separator);
    os.writeBytes("Table");
    os.write(separator);
    os.writeBytes("Partition");
    os.write(separator);
    os.writeBytes("State");
    os.write(separator);
    os.writeBytes("Blocked By");
    os.write(separator);
    os.writeBytes("Type");
    os.write(separator);
    os.writeBytes("Transaction ID");
    os.write(separator);
    os.writeBytes("Last Heartbeat");
    os.write(separator);
    os.writeBytes("Acquired At");
    os.write(separator);
    os.writeBytes("User");
    os.write(separator);
    os.writeBytes("Hostname");
    os.write(separator);
    os.writeBytes("Agent Info");
    os.write(terminator);
    List<ShowLocksResponseElement> locks = rsp.getLocks();
    if (locks != null) {
        for (ShowLocksResponseElement lock : locks) {
            if (lock.isSetLockIdInternal()) {
                os.writeBytes(Long.toString(lock.getLockid()) + "." + Long.toString(lock.getLockIdInternal()));
            } else {
                os.writeBytes(Long.toString(lock.getLockid()));
            }
            os.write(separator);
            os.writeBytes(lock.getDbname());
            os.write(separator);
            os.writeBytes((lock.getTablename() == null) ? "NULL" : lock.getTablename());
            os.write(separator);
            os.writeBytes((lock.getPartname() == null) ? "NULL" : lock.getPartname());
            os.write(separator);
            os.writeBytes(lock.getState().toString());
            os.write(separator);
            if (lock.isSetBlockedByExtId()) {
                //both "blockedby" are either there or not
                os.writeBytes(Long.toString(lock.getBlockedByExtId()) + "." + Long.toString(lock.getBlockedByIntId()));
            } else {
                //12 chars - try to keep cols aligned
                os.writeBytes("            ");
            }
            os.write(separator);
            os.writeBytes(lock.getType().toString());
            os.write(separator);
            os.writeBytes((lock.getTxnid() == 0) ? "NULL" : Long.toString(lock.getTxnid()));
            os.write(separator);
            os.writeBytes(Long.toString(lock.getLastheartbeat()));
            os.write(separator);
            os.writeBytes((lock.getAcquiredat() == 0) ? "NULL" : Long.toString(lock.getAcquiredat()));
            os.write(separator);
            os.writeBytes(lock.getUser());
            os.write(separator);
            os.writeBytes(lock.getHostname());
            os.write(separator);
            os.writeBytes(lock.getAgentInfo() == null ? "NULL" : lock.getAgentInfo());
            os.write(separator);
            os.write(terminator);
        }
    }
}