Examples with AllocateTableWriteIdsResponse org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse used on opensource projects

Example 1 with AllocateTableWriteIdsResponse

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse in project hive by apache.

the class TxnHandler method allocateTableWriteIds.

@Override
public AllocateTableWriteIdsResponse allocateTableWriteIds(AllocateTableWriteIdsRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
    List<Long> txnIds = rqst.getTxnIds();
    String dbName = rqst.getDbName().toLowerCase();
    String tblName = rqst.getTableName().toLowerCase();
    try {
        Connection dbConn = null;
        Statement stmt = null;
        ResultSet rs = null;
        TxnStore.MutexAPI.LockHandle handle = null;
        try {
            lockInternal();
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            // easier to read logs
            Collections.sort(txnIds);
            // Check if all the input txns are in open state. Write ID should be allocated only for open transactions.
            if (!isTxnsInOpenState(txnIds, stmt)) {
                ensureAllTxnsValid(dbName, tblName, txnIds, stmt);
                throw new RuntimeException("This should never happen for txnIds: " + txnIds);
            }
            List<TxnToWriteId> txnToWriteIds = new ArrayList<>();
            List<Long> allocatedTxns = new ArrayList<>();
            long txnId;
            long writeId;
            List<String> queries = new ArrayList<>();
            StringBuilder prefix = new StringBuilder();
            StringBuilder suffix = new StringBuilder();
            // Traverse the TXN_TO_WRITE_ID to see if any of the input txns already have allocated a
            // write id for the same db.table. If yes, then need to reuse it else have to allocate new one
            // The write id would have been already allocated in case of multi-statement txns where
            // first write on a table will allocate write id and rest of the writes should re-use it.
            prefix.append("select t2w_txnid, t2w_writeid from TXN_TO_WRITE_ID where" + " t2w_database = " + quoteString(dbName) + " and t2w_table = " + quoteString(tblName) + " and ");
            suffix.append("");
            TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnIds, "t2w_txnid", false, false);
            for (String query : queries) {
                LOG.debug("Going to execute query <" + query + ">");
                rs = stmt.executeQuery(query);
                while (rs.next()) {
                    // If table write ID is already allocated for the given transaction, then just use it
                    txnId = rs.getLong(1);
                    writeId = rs.getLong(2);
                    txnToWriteIds.add(new TxnToWriteId(txnId, writeId));
                    allocatedTxns.add(txnId);
                    LOG.info("Reused already allocated writeID: " + writeId + " for txnId: " + txnId);
                }
            }
            // If all the txns in the list have already allocated write ids, then just skip new allocations
            long numOfWriteIds = txnIds.size() - allocatedTxns.size();
            assert (numOfWriteIds >= 0);
            if (0 == numOfWriteIds) {
                // If all the txns in the list have pre-allocated write ids for the given table, then just return
                return new AllocateTableWriteIdsResponse(txnToWriteIds);
            }
            handle = getMutexAPI().acquireLock(MUTEX_KEY.WriteIdAllocator.name());
            // There are some txns in the list which has no write id allocated and hence go ahead and do it.
            // Get the next write id for the given table and update it with new next write id.
            // This is select for update query which takes a lock if the table entry is already there in NEXT_WRITE_ID
            String s = sqlGenerator.addForUpdateClause("select nwi_next from NEXT_WRITE_ID where nwi_database = " + quoteString(dbName) + " and nwi_table = " + quoteString(tblName));
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            if (!rs.next()) {
                // First allocation of write id should add the table to the next_write_id meta table
                // The initial value for write id should be 1 and hence we add 1 with number of write ids allocated here
                s = "insert into NEXT_WRITE_ID (nwi_database, nwi_table, nwi_next) values (" + quoteString(dbName) + "," + quoteString(tblName) + "," + String.valueOf(numOfWriteIds + 1) + ")";
                LOG.debug("Going to execute insert <" + s + ">");
                stmt.execute(s);
                writeId = 1;
            } else {
                // Update the NEXT_WRITE_ID for the given table after incrementing by number of write ids allocated
                writeId = rs.getLong(1);
                s = "update NEXT_WRITE_ID set nwi_next = " + (writeId + numOfWriteIds) + " where nwi_database = " + quoteString(dbName) + " and nwi_table = " + quoteString(tblName);
                LOG.debug("Going to execute update <" + s + ">");
                stmt.executeUpdate(s);
            }
            // Map the newly allocated write ids against the list of txns which doesn't have pre-allocated
            // write ids
            List<String> rows = new ArrayList<>();
            for (long txn : txnIds) {
                if (allocatedTxns.contains(txn)) {
                    continue;
                }
                rows.add(txn + ", " + quoteString(dbName) + ", " + quoteString(tblName) + ", " + writeId);
                txnToWriteIds.add(new TxnToWriteId(txn, writeId));
                LOG.info("Allocated writeID: " + writeId + " for txnId: " + txn);
                writeId++;
            }
            // Insert entries to TXN_TO_WRITE_ID for newly allocated write ids
            List<String> inserts = sqlGenerator.createInsertValuesStmt("TXN_TO_WRITE_ID (t2w_txnid, t2w_database, t2w_table, t2w_writeid)", rows);
            for (String insert : inserts) {
                LOG.debug("Going to execute insert <" + insert + ">");
                stmt.execute(insert);
            }
            LOG.debug("Going to commit");
            dbConn.commit();
            return new AllocateTableWriteIdsResponse(txnToWriteIds);
        } catch (SQLException e) {
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "allocateTableWriteIds(" + rqst + ")");
            throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e));
        } finally {
            close(rs, stmt, dbConn);
            if (handle != null) {
                handle.releaseLocks();
            }
            unlockInternal();
        }
    } catch (RetryException e) {
        return allocateTableWriteIds(rqst);
    }
}

Example 2 with AllocateTableWriteIdsResponse

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse in project hive by apache.

the class TestCompactionTxnHandler method addDynamicPartitions.

@Test
public void addDynamicPartitions() throws Exception {
    String dbName = "default";
    String tableName = "adp_table";
    OpenTxnsResponse openTxns = txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
    long txnId = openTxns.getTxn_ids().get(0);
    AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(openTxns.getTxn_ids(), dbName, tableName));
    long writeId = writeIds.getTxnToWriteIds().get(0).getWriteId();
    assertEquals(txnId, writeIds.getTxnToWriteIds().get(0).getTxnId());
    assertEquals(1, writeId);
    // lock a table, as in dynamic partitions
    LockComponent lc = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, dbName);
    lc.setIsDynamicPartitionWrite(true);
    lc.setTablename(tableName);
    DataOperationType dop = DataOperationType.UPDATE;
    lc.setOperationType(dop);
    LockRequest lr = new LockRequest(Arrays.asList(lc), "me", "localhost");
    lr.setTxnid(txnId);
    LockResponse lock = txnHandler.lock(lr);
    assertEquals(LockState.ACQUIRED, lock.getState());
    AddDynamicPartitions adp = new AddDynamicPartitions(txnId, writeId, dbName, tableName, Arrays.asList("ds=yesterday", "ds=today"));
    adp.setOperationType(dop);
    txnHandler.addDynamicPartitions(adp);
    txnHandler.commitTxn(new CommitTxnRequest(txnId));
    Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(1000);
    assertEquals(2, potentials.size());
    SortedSet<CompactionInfo> sorted = new TreeSet<CompactionInfo>(potentials);
    int i = 0;
    for (CompactionInfo ci : sorted) {
        assertEquals(dbName, ci.dbname);
        assertEquals(tableName, ci.tableName);
        switch(i++) {
            case 0:
                assertEquals("ds=today", ci.partName);
                break;
            case 1:
                assertEquals("ds=yesterday", ci.partName);
                break;
            default:
                throw new RuntimeException("What?");
        }
    }
}

Example 3 with AllocateTableWriteIdsResponse

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse in project hive by apache.

the class TestTxnHandler method testAbortTxn.

@Test
public void testAbortTxn() throws Exception {
    OpenTxnsResponse openedTxns = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    List<Long> txnList = openedTxns.getTxn_ids();
    long first = txnList.get(0);
    assertEquals(1L, first);
    long second = txnList.get(1);
    assertEquals(2L, second);
    txnHandler.abortTxn(new AbortTxnRequest(1));
    List<String> parts = new ArrayList<String>();
    parts.add("p=1");
    AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(Collections.singletonList(3L), "default", "T"));
    long writeId = writeIds.getTxnToWriteIds().get(0).getWriteId();
    assertEquals(3, writeIds.getTxnToWriteIds().get(0).getTxnId());
    assertEquals(1, writeId);
    AddDynamicPartitions adp = new AddDynamicPartitions(3, writeId, "default", "T", parts);
    adp.setOperationType(DataOperationType.INSERT);
    txnHandler.addDynamicPartitions(adp);
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(3, txnsInfo.getTxn_high_water_mark());
    assertEquals(3, txnsInfo.getOpen_txns().size());
    assertEquals(1L, txnsInfo.getOpen_txns().get(0).getId());
    assertEquals(TxnState.ABORTED, txnsInfo.getOpen_txns().get(0).getState());
    assertEquals(2L, txnsInfo.getOpen_txns().get(1).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(1).getState());
    assertEquals(3, txnsInfo.getOpen_txns().get(2).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(2).getState());
    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(3, txns.getTxn_high_water_mark());
    assertEquals(3, txns.getOpen_txns().size());
    boolean[] saw = new boolean[4];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (Long tid : txns.getOpen_txns()) {
        saw[tid.intValue()] = true;
    }
    for (int i = 1; i < saw.length; i++) assertTrue(saw[i]);
    txnHandler.commitTxn(new CommitTxnRequest(2));
    // this succeeds as abortTxn is idempotent
    txnHandler.abortTxn(new AbortTxnRequest(1));
    boolean gotException = false;
    try {
        txnHandler.abortTxn(new AbortTxnRequest(2));
    } catch (NoSuchTxnException ex) {
        gotException = true;
        // if this wasn't an empty txn, we'd get a better msg
        Assert.assertEquals("No such transaction " + JavaUtils.txnIdToString(2), ex.getMessage());
    }
    Assert.assertTrue(gotException);
    gotException = false;
    txnHandler.commitTxn(new CommitTxnRequest(3));
    try {
        txnHandler.abortTxn(new AbortTxnRequest(3));
    } catch (NoSuchTxnException ex) {
        gotException = true;
        // txn 3 is not empty txn, so we get a better msg
        Assert.assertEquals("Transaction " + JavaUtils.txnIdToString(3) + " is already committed.", ex.getMessage());
    }
    Assert.assertTrue(gotException);
    gotException = false;
    try {
        txnHandler.abortTxn(new AbortTxnRequest(4));
    } catch (NoSuchTxnException ex) {
        gotException = true;
        Assert.assertEquals("No such transaction " + JavaUtils.txnIdToString(4), ex.getMessage());
    }
    Assert.assertTrue(gotException);
}

Example 4 with AllocateTableWriteIdsResponse

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking4.

/**
 * txns overlap, update same resource, simulate multi-stmt txn case
 * Also tests that we kill txn when it tries to acquire lock if we already know it will not be committed
 */
@Test
public void testWriteSetTracking4() throws Exception {
    dropTable(new String[] { "TAB_PART", "TAB2" });
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    txnMgr.openTxn(ctx, "Long Running");
    checkCmdOnDriver(driver.compileAndRespond("select a from  TAB_PART where p = 'blah'"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    // for some reason this just locks the table; if I alter table to add this partition, then
    // we end up locking both table and partition with share_read.  (Plan has 2 ReadEntities)...?
    // same for other locks below
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr2.openTxn(ctx, "Short Running");
    // no such partition
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'blah'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "Short Running");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    // update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(Collections.singletonList(txnMgr2.getCurrentTxnId()), "default", "tab2"));
    Assert.assertEquals(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    // Short Running updated nothing, so we expect 0 rows in WRITE_SET
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    txnMgr2.openTxn(ctx, "T3");
    // pretend this partition exists
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'"));
    txnMgr2.acquireLocks(driver.getPlan(), ctx, "T3");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
    // since TAB2 is empty
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
    // update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(Collections.singletonList(txnMgr2.getCurrentTxnId()), "default", "tab2"));
    Assert.assertEquals(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.singletonList("p=two"));
    adp.setOperationType(DataOperationType.UPDATE);
    // simulate partition update
    txnHandler.addDynamicPartitions(adp);
    txnMgr2.commitTxn();
    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    AcidWriteSetService houseKeeper = new AcidWriteSetService();
    houseKeeper.setConf(conf);
    houseKeeper.run();
    // since T3 overlaps with Long Running (still open) GC does nothing
    Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    // no rows match
    checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
    writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(Collections.singletonList(txnMgr.getCurrentTxnId()), "default", "tab2"));
    Assert.assertEquals(txnMgr.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getTxnId());
    // so generate empty Dyn Part call
    adp = new AddDynamicPartitions(txnMgr.getCurrentTxnId(), writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "tab2", Collections.EMPTY_LIST);
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    txnMgr.commitTxn();
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 0, locks.size());
    houseKeeper.run();
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
}

Example 5 with AllocateTableWriteIdsResponse

use of org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse in project hive by apache.

the class TestDbTxnManager2 method testWriteSetTracking5.

/**
 * overlapping txns updating the same resource but 1st one rolls back; 2nd commits
 * @throws Exception
 */
@Test
public void testWriteSetTracking5() throws Exception {
    dropTable(new String[] { "TAB_PART" });
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
    checkCmdOnDriver(cpr);
    checkCmdOnDriver(driver.run("insert into TAB_PART partition(p='blah') values(1,2)"));
    HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    txnMgr.openTxn(ctx, "Known");
    long txnId = txnMgr2.openTxn(ctx, "Unknown");
    checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Known");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", "p=blah", locks);
    checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'"));
    ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Unknown", false);
    // should not matter which txnMgr is used here
    locks = getLocks(txnMgr2);
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", "p=blah", locks);
    checkLock(LockType.SHARED_WRITE, LockState.WAITING, "default", "TAB_PART", "p=blah", locks);
    txnMgr.rollbackTxn();
    AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(new AllocateTableWriteIdsRequest(Collections.singletonList(txnId), "default", "TAB_PART"));
    Assert.assertEquals(txnId, writeIds.getTxnToWriteIds().get(0).getTxnId());
    AddDynamicPartitions adp = new AddDynamicPartitions(txnId, writeIds.getTxnToWriteIds().get(0).getWriteId(), "default", "TAB_PART", Arrays.asList("p=blah"));
    adp.setOperationType(DataOperationType.UPDATE);
    txnHandler.addDynamicPartitions(adp);
    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
    // since conflicting txn rolled back, commit succeeds
    txnMgr2.commitTxn();
    Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
}