Examples with ShowLocksResponseElement org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement used on opensource projects

Example 16 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking2.

/**
   * txns overlap in time but do not update same resource - no conflict
   */
@Test
public void testWriteSetTracking2() throws Exception {
    dropTable(new String[] { "TAB_PART", "TAB2" });
    CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr.openTxn(ctx, "Peter");
    checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Peter");
    txnMgr2.openTxn(ctx, "Catherine");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    //note that "update" uses dynamic partitioning thus lock is on the table not partition
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    txnMgr.commitTxn();
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 9 where p = 'doh'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "Catherine");
    txnMgr2.commitTxn();
}

Example 17 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking7.

/**
   * 2 concurrent txns update different partitions of the same table and succeed
   * @throws Exception
   */
@Test
public void testWriteSetTracking7() throws Exception {
    dropTable(new String[] { "tab2", "TAB2" });
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
    CommandProcessorResponse cpr = driver.run("create table if not exists tab2 (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    //txnid:1
    checkCmdOnDriver(driver.run("insert into tab2 partition(p)(a,b,p) values(1,1,'one'),(2,2,'two')"));
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    //test with predicates such that partition pruning works
    txnMgr2.openTxn(ctx, "T2");
    checkCmdOnDriver(driver.compileAndRespond("update tab2 set b = 7 where p='two'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T2");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", "p=two", locks);
    //now start concurrent txn
    txnMgr.openTxn(ctx, "T3");
    checkCmdOnDriver(driver.compileAndRespond("update tab2 set b = 7 where p='one'"));
    ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "T3", false);
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", "p=one", locks);
    //this simulates the completion of txnid:2
    AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab2", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    //txnid:2
    txnMgr2.commitTxn();
    locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", "p=one", locks);
    //completion of txnid:3
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), "default", "tab2", Collections.singletonList("p=one"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    //txnid:3
    txnMgr.commitTxn();
    //now both txns concurrently updated TAB2 but different partitions.
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'"));
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'"));
    //2 from txnid:1, 1 from txnid:2, 1 from txnid:3
    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null"));
    //================
    //test with predicates such that partition pruning doesn't kick in
    cpr = driver.run("create table if not exists tab1 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    //txnid:4
    checkCmdOnDriver(driver.run("insert into tab1 partition(p)(a,b,p) values(1,1,'one'),(2,2,'two')"));
    txnMgr2.openTxn(ctx, "T5");
    checkCmdOnDriver(driver.compileAndRespond("update tab1 set b = 7 where b=1"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T5");
    locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=one", locks);
    //now start concurrent txn
    txnMgr.openTxn(ctx, "T6");
    checkCmdOnDriver(driver.compileAndRespond("update tab1 set b = 7 where b = 2"));
    ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "T6", false);
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 4, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=one", locks);
    checkLock(LockType.SHARED_WRITE, LockState.WAITING, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.WAITING, "default", "TAB1", "p=one", locks);
    //this simulates the completion of txnid:5
    adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab1", Collections.singletonList("p=one"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    //txnid:5
    txnMgr2.commitTxn();
    //retest WAITING locks (both have same ext id)
    ((DbLockManager) txnMgr.getLockManager()).checkLock(locks.get(2).getLockid());
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=one", locks);
    //completion of txnid:6
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), "default", "tab1", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    //txnid:6
    txnMgr.commitTxn();
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
    //2 from insert + 1 for each update stmt
    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}

Example 18 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method checkExpectedLocks.

/**
   * collection of queries where we ensure that we get the locks that are expected
   * @throws Exception
   */
@Test
public void checkExpectedLocks() throws Exception {
    dropTable(new String[] { "acidPart", "nonAcidPart" });
    CommandProcessorResponse cpr = null;
    cpr = driver.run("create table acidPart(a int, b int) partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("insert into nonAcidPart partition(p) values(1,2,3)");
    checkCmdOnDriver(cpr);
    LockState lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", null, locks);
    List<HiveLock> relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
    cpr = driver.compileAndRespond("insert into nonAcidPart partition(p=1) values(5,6)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", "p=1", locks);
    relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
    cpr = driver.compileAndRespond("insert into acidPart partition(p) values(1,2,3)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", null, locks);
    relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
    cpr = driver.compileAndRespond("insert into acidPart partition(p=1) values(5,6)");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", "p=1", locks);
    relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
    cpr = driver.compileAndRespond("update acidPart set b = 17 where a = 1");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "acidPart", null, locks);
    relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
    cpr = driver.compileAndRespond("update acidPart set b = 17 where p = 1");
    checkCmdOnDriver(cpr);
    lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    //https://issues.apache.org/jira/browse/HIVE-13212
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "acidPart", null, locks);
    relLocks = new ArrayList<HiveLock>(1);
    relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
    txnMgr.getLockManager().releaseLocks(relLocks);
}

Example 19 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking10.

/**
   * Concurrent update/delete of same partition - should fail to commit
   */
@Test
public void testWriteSetTracking10() throws Exception {
    dropTable(new String[] { "TAB1" });
    CommandProcessorResponse cpr = driver.run("create table if not exists tab1 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    //txnid:1
    checkCmdOnDriver(driver.run("insert into tab1 partition(p)(a,b,p) values(1,1,'one'),(2,2,'two')"));
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "T2");
    checkCmdOnDriver(driver.compileAndRespond("update tab1 set b = 7 where b=2"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T2");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=one", locks);
    //now start concurrent txn
    txnMgr.openTxn(ctx, "T3");
    checkCmdOnDriver(driver.compileAndRespond("delete from tab1 where p='two' and b=2"));
    ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "T3", false);
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 3, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=one", locks);
    checkLock(LockType.SHARED_WRITE, LockState.WAITING, "default", "TAB1", "p=two", locks);
    //this simulates the completion of txnid:2
    AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab1", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    //txnid:2
    txnMgr2.commitTxn();
    //retest WAITING locks (both have same ext id)
    ((DbLockManager) txnMgr.getLockManager()).checkLock(locks.get(2).getLockid());
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB1", "p=two", locks);
    //completion of txnid:3
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), "default", "tab1", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.DELETE);
    txnHandler.addDynamicPartitions(adp);
    LockException exception = null;
    try {
        //txnid:3
        txnMgr.commitTxn();
    } catch (LockException e) {
        exception = e;
    }
    Assert.assertNotEquals("Expected exception", null, exception);
    Assert.assertEquals("Exception msg doesn't match", "Aborting [txnid:3,3] due to a write conflict on default/tab1/p=two committed by [txnid:2,3] d/u", exception.getCause().getMessage());
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), 3, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}

Example 20 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testShowTablesLock.

@Test
public void testShowTablesLock() throws Exception {
    dropTable(new String[] { "T, T2" });
    CommandProcessorResponse cpr = driver.run("create table if not exists T (a int, b int)");
    checkCmdOnDriver(cpr);
    long txnid1 = txnMgr.openTxn(ctx, "Fifer");
    checkCmdOnDriver(driver.compileAndRespond("insert into T values(1,3)"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t", null, locks);
    DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    checkCmdOnDriver(driver.compileAndRespond("show tables"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "Fidler");
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t", null, locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks);
    txnMgr.commitTxn();
    txnMgr2.releaseLocks(txnMgr2.getLockManager().getLocks(false, false));
    Assert.assertEquals("Lock remained", 0, getLocks().size());
    Assert.assertEquals("Lock remained", 0, getLocks(txnMgr2).size());
    cpr = driver.run("create table if not exists T2 (a int, b int) partitioned by (p int) clustered by (a) " + "into 2  buckets stored as orc TBLPROPERTIES ('transactional'='false')");
    checkCmdOnDriver(cpr);
    txnid1 = txnMgr.openTxn(ctx, "Fifer");
    checkCmdOnDriver(driver.compileAndRespond("insert into T2 partition(p=1) values(1,3)"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t2", "p=1", locks);
    txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    checkCmdOnDriver(driver.compileAndRespond("show tables"));
    ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fidler", false);
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t2", "p=1", locks);
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks);
    txnMgr.commitTxn();
    txnMgr2.releaseLocks(txnMgr2.getLockManager().getLocks(false, false));
    Assert.assertEquals("Lock remained", 0, getLocks().size());
    Assert.assertEquals("Lock remained", 0, getLocks(txnMgr2).size());
}