Examples with AllocateTableWriteIdsRequest org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest used on opensource projects

Example 11 with AllocateTableWriteIdsRequest

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking4.

/**
 * txns overlap, update same resource, simulate multi-stmt txn case
 * Also tests that we kill txn when it tries to acquire lock if we already know it will not be committed
 */
@Test
public void testWriteSetTracking4() throws Exception {
    dropTable(new String[] { "TAB_PART", "TAB2" });
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    txnMgr.openTxn(ctx, "Long Running");
    driver.compileAndRespond("select a from TAB_PART where p = 'blah'", true);
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    // for some reason this just locks the table; if I alter table to add this partition, then
    // we end up locking both table and partition with share_read.  (Plan has 2 ReadEntities)...?
    // same for other locks below
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "Short Running");
    // no such partition
    driver.compileAndRespond("update TAB2 set b = 7 where p = 'blah'", true);
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "Short Running");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    // update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    AllocateTableWriteIdsRequest rqst = new AllocateTableWriteIdsRequest("default", "tab2");
    rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId()));
    AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(rqst);
    Assert.assertEquals(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    // Short Running updated nothing, so we expect 0 rows in WRITE_SET
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    txnMgr2.openTxn(ctx, "T3");
    // pretend this partition exists
    driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'", true);
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T3");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    // since TAB2 is empty
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    // update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    rqst = new AllocateTableWriteIdsRequest("default", "tab2");
    rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId()));
    writeIds = txnHandler.allocateTableWriteIds(rqst);
    Assert.assertEquals(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    // simulate partition update
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    Assert.assertEquals("WRITE_SET mismatch: " + TestTxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    MetastoreTaskThread houseKeeper = new AcidHouseKeeperService();
    houseKeeper.setConf(conf);
    houseKeeper.run();
    // since T3 overlaps with Long Running (still open) GC does nothing
    Assert.assertEquals(1, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
    // no rows match
    driver.compileAndRespond("update TAB2 set b = 17 where a = 1", true);
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    rqst = new AllocateTableWriteIdsRequest("default", "tab2");
    rqst.setTxnIds(Collections.singletonList(txnMgr.getCurrentTxnId()));
    writeIds = txnHandler.allocateTableWriteIds(rqst);
    Assert.assertEquals(txnMgr.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    // so generate empty Dyn Part call
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr.commitTxn();
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 0, locks.size());
    /*
      The last transaction will always remain in the transaction table, so we will open an other one,
      wait for the timeout period to exceed, then start the initiator that will clean
     */
    txnMgr.openTxn(ctx, "Long Running");
    Thread.sleep(txnHandler.getOpenTxnTimeOutMillis());
    // Now we can clean the write_set
    houseKeeper.run();
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\""));
}

Example 12 with AllocateTableWriteIdsRequest

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest in project hive by apache.

the class BaseReplicationScenariosAcidTables method allocateWriteIdsForTablesAndAcquireLocks.

List<Long> allocateWriteIdsForTablesAndAcquireLocks(String primaryDbName, Map<String, Long> tables, TxnStore txnHandler, List<Long> txns, HiveConf primaryConf) throws Throwable {
    AllocateTableWriteIdsRequest rqst = new AllocateTableWriteIdsRequest();
    rqst.setDbName(primaryDbName);
    List<Long> lockIds = new ArrayList<>();
    for (Map.Entry<String, Long> entry : tables.entrySet()) {
        rqst.setTableName(entry.getKey());
        rqst.setTxnIds(txns);
        txnHandler.allocateTableWriteIds(rqst);
        for (long txnId : txns) {
            LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, primaryDbName);
            comp.setTablename(entry.getKey());
            comp.setOperationType(DataOperationType.UPDATE);
            List<LockComponent> components = new ArrayList<LockComponent>(1);
            components.add(comp);
            LockRequest lockRequest = new LockRequest(components, "u1", "hostname");
            lockRequest.setTxnid(txnId);
            lockIds.add(txnHandler.lock(lockRequest).getLockid());
        }
    }
    verifyWriteIdsForTables(tables, primaryConf, primaryDbName);
    return lockIds;
}

Example 13 with AllocateTableWriteIdsRequest

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest in project hive by apache.

the class CompactorTest method burnThroughTransactions.

protected void burnThroughTransactions(String dbName, String tblName, int num, Set<Long> open, Set<Long> aborted) throws MetaException, NoSuchTxnException, TxnAbortedException {
    OpenTxnsResponse rsp = txnHandler.openTxns(new OpenTxnRequest(num, "me", "localhost"));
    AllocateTableWriteIdsRequest awiRqst = new AllocateTableWriteIdsRequest(rsp.getTxn_ids(), dbName, tblName);
    AllocateTableWriteIdsResponse awiResp = txnHandler.allocateTableWriteIds(awiRqst);
    int i = 0;
    for (long tid : rsp.getTxn_ids()) {
        assert (awiResp.getTxnToWriteIds().get(i++).getTxnId() == tid);
        if (aborted != null && aborted.contains(tid)) {
            txnHandler.abortTxn(new AbortTxnRequest(tid));
        } else if (open == null || (open != null && !open.contains(tid))) {
            txnHandler.commitTxn(new CommitTxnRequest(tid));
        }
    }
}

Example 14 with AllocateTableWriteIdsRequest

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest in project hetu-core by openlookeng.

the class ThriftHiveMetastoreClient method getTableWriteId.

@Override
public long getTableWriteId(String dbName, String tableName, long transactionId) throws TException {
    AllocateTableWriteIdsRequest allocateTableWriteIdsRequest = new AllocateTableWriteIdsRequest();
    allocateTableWriteIdsRequest.setDbName(dbName);
    allocateTableWriteIdsRequest.setTableName(tableName);
    allocateTableWriteIdsRequest.addToTxnIds(transactionId);
    final AllocateTableWriteIdsResponse allocateTableWriteIdsResponse = client.allocate_table_write_ids(allocateTableWriteIdsRequest);
    long txnToWriteId = allocateTableWriteIdsResponse.getTxnToWriteIds().stream().filter(e -> e.getTxnId() == transactionId).collect(toImmutableList()).get(0).getWriteId();
    return txnToWriteId;
}