Examples with LockResponse org.apache.hadoop.hive.metastore.api.LockResponse used on opensource projects

Example 61 with LockResponse

use of org.apache.hadoop.hive.metastore.api.LockResponse in project hive by apache.

the class TestTxnHandler method testLockEESR.

@Test
public void testLockEESR() throws Exception {
    // Test that exclusive blocks exclusive and read
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
    comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.SELECT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.WAITING);
}

Example 62 with LockResponse

use of org.apache.hadoop.hive.metastore.api.LockResponse in project hive by apache.

the class TestTxnHandler method testCheckLockTxnAborted.

@Test
public void testCheckLockTxnAborted() throws Exception {
    // Test that when a transaction is aborted, the heartbeat fails
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    long lockid = res.getLockid();
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    try {
        // This will throw NoSuchLockException (even though it's the
        // transaction we've closed) because that will have deleted the lock.
        txnHandler.checkLock(new CheckLockRequest(lockid));
        fail("Allowed to check lock on aborted transaction.");
    } catch (NoSuchLockException e) {
    }
}

Example 63 with LockResponse

use of org.apache.hadoop.hive.metastore.api.LockResponse in project hive by apache.

the class TestTxnHandler method testMultipleLock.

@Test
public void testMultipleLock() throws Exception {
    // Test more than one lock can be handled in a lock request
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(2);
    components.add(comp);
    comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("anotherpartition=anothervalue");
    comp.setOperationType(DataOperationType.NO_TXN);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    long lockid = res.getLockid();
    assertTrue(res.getState() == LockState.ACQUIRED);
    res = txnHandler.checkLock(new CheckLockRequest(lockid));
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.unlock(new UnlockRequest(lockid));
    assertEquals(0, txnHandler.numLocksInLockTable());
}

Example 64 with LockResponse

use of org.apache.hadoop.hive.metastore.api.LockResponse in project hive by apache.

the class TestTxnHandler method testCheckLockAcquireAfterWaiting.

@Test
public void testCheckLockAcquireAfterWaiting() throws Exception {
    LockComponent comp = new LockComponent(LockType.EXCL_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    long txnId = openTxn();
    req.setTxnid(txnId);
    LockResponse res = txnHandler.lock(req);
    long lockid1 = res.getLockid();
    assertTrue(res.getState() == LockState.ACQUIRED);
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setPartitionname("mypartition=myvalue");
    comp.setOperationType(DataOperationType.INSERT);
    components.clear();
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(openTxn());
    res = txnHandler.lock(req);
    long lockid2 = res.getLockid();
    assertTrue(res.getState() == LockState.WAITING);
    txnHandler.abortTxn(new AbortTxnRequest(txnId));
    res = txnHandler.checkLock(new CheckLockRequest(lockid2));
    assertTrue(res.getState() == LockState.ACQUIRED);
}

Example 65 with LockResponse

use of org.apache.hadoop.hive.metastore.api.LockResponse in project hive by apache.

the class Cleaner method removeFiles.

private boolean removeFiles(String location, long minOpenTxnGLB, CompactionInfo ci, boolean dropPartition) throws MetaException, IOException, NoSuchObjectException, NoSuchTxnException {
    if (dropPartition) {
        LockRequest lockRequest = createLockRequest(ci, 0, LockType.EXCL_WRITE, DataOperationType.DELETE);
        LockResponse res = null;
        try {
            res = txnHandler.lock(lockRequest);
            if (res.getState() == LockState.ACQUIRED) {
                // check if partition wasn't recreated
                if (resolvePartition(ci) == null) {
                    return removeFiles(location, ci);
                }
            }
        } catch (NoSuchTxnException | TxnAbortedException e) {
            LOG.error(e.getMessage());
        } finally {
            if (res != null && res.getState() != LockState.NOT_ACQUIRED) {
                try {
                    txnHandler.unlock(new UnlockRequest(res.getLockid()));
                } catch (NoSuchLockException | TxnOpenException e) {
                    LOG.error(e.getMessage());
                }
            }
        }
    }
    ValidTxnList validTxnList = TxnUtils.createValidTxnListForCleaner(txnHandler.getOpenTxns(), minOpenTxnGLB);
    // save it so that getAcidState() sees it
    conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString());
    /**
     * {@code validTxnList} is capped by minOpenTxnGLB so if
     * {@link AcidUtils#getAcidState(Path, Configuration, ValidWriteIdList)} sees a base/delta
     * produced by a compactor, that means every reader that could be active right now see it
     * as well.  That means if this base/delta shadows some earlier base/delta, the it will be
     * used in favor of any files that it shadows.  Thus the shadowed files are safe to delete.
     *
     * The metadata about aborted writeIds (and consequently aborted txn IDs) cannot be deleted
     * above COMPACTION_QUEUE.CQ_HIGHEST_WRITE_ID.
     * See {@link TxnStore#markCleaned(CompactionInfo)} for details.
     * For example given partition P1, txnid:150 starts and sees txnid:149 as open.
     * Say compactor runs in txnid:160, but 149 is still open and P1 has the largest resolved
     * writeId:17.  Compactor will produce base_17_c160.
     * Suppose txnid:149 writes delta_18_18
     * to P1 and aborts.  Compactor can only remove TXN_COMPONENTS entries
     * up to (inclusive) writeId:17 since delta_18_18 may be on disk (and perhaps corrupted) but
     * not visible based on 'validTxnList' capped at minOpenTxn so it will not not be cleaned by
     * {@link #removeFiles(String, ValidWriteIdList, CompactionInfo)} and so we must keep the
     * metadata that says that 18 is aborted.
     * In a slightly different case, whatever txn created delta_18 (and all other txn) may have
     * committed by the time cleaner runs and so cleaner will indeed see delta_18_18 and remove
     * it (since it has nothing but aborted data).  But we can't tell which actually happened
     * in markCleaned() so make sure it doesn't delete meta above CG_CQ_HIGHEST_WRITE_ID.
     *
     * We could perhaps make cleaning of aborted and obsolete and remove all aborted files up
     * to the current Min Open Write Id, this way aborted TXN_COMPONENTS meta can be removed
     * as well up to that point which may be higher than CQ_HIGHEST_WRITE_ID.  This could be
     * useful if there is all of a sudden a flood of aborted txns.  (For another day).
     */
    // Creating 'reader' list since we are interested in the set of 'obsolete' files
    ValidReaderWriteIdList validWriteIdList = getValidCleanerWriteIdList(ci, validTxnList);
    LOG.debug("Cleaning based on writeIdList: {}", validWriteIdList);
    return removeFiles(location, validWriteIdList, ci);
}