Examples with OpenTxnRequest org.apache.hadoop.hive.metastore.api.OpenTxnRequest used on opensource projects

Example 16 with OpenTxnRequest

use of org.apache.hadoop.hive.metastore.api.OpenTxnRequest in project hive by apache.

the class TestCompactionTxnHandler method testMarkCleanedCleansTxnsAndTxnComponents.

// TODO test changes to mark cleaned to clean txns and txn_components
@Test
public void testMarkCleanedCleansTxnsAndTxnComponents() throws Exception {
    long txnid = openTxn();
    long mytableWriteId = allocateTableWriteIds("mydb", "mytable", txnid);
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.INSERT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    txnid = openTxn();
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("yourtable");
    comp.setOperationType(DataOperationType.DELETE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    txnid = openTxn();
    long fooWriteId = allocateTableWriteIds("mydb", "foo", txnid);
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("foo");
    comp.setPartitionname("bar=compact");
    comp.setOperationType(DataOperationType.UPDATE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("foo");
    comp.setPartitionname("baz=compact");
    comp.setOperationType(DataOperationType.UPDATE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    CompactionInfo ci;
    // Now clean them and check that they are removed from the count.
    CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MAJOR);
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean(0, 0).size());
    ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    ci.highestWriteId = mytableWriteId;
    txnHandler.updateCompactorState(ci, 0);
    txnHandler.markCompacted(ci);
    Thread.sleep(txnHandler.getOpenTxnTimeOutMillis());
    List<CompactionInfo> toClean = txnHandler.findReadyToClean(0, 0);
    assertEquals(1, toClean.size());
    txnHandler.markCleaned(ci);
    // Check that we are cleaning up the empty aborted transactions
    GetOpenTxnsResponse txnList = txnHandler.getOpenTxns();
    assertEquals(3, txnList.getOpen_txnsSize());
    // Create one aborted for low water mark
    txnid = openTxn();
    txnHandler.abortTxn(new AbortTxnRequest(txnid));
    Thread.sleep(txnHandler.getOpenTxnTimeOutMillis());
    txnHandler.cleanEmptyAbortedAndCommittedTxns();
    txnList = txnHandler.getOpenTxns();
    assertEquals(3, txnList.getOpen_txnsSize());
    rqst = new CompactionRequest("mydb", "foo", CompactionType.MAJOR);
    rqst.setPartitionname("bar");
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean(0, 0).size());
    ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    ci.highestWriteId = fooWriteId;
    txnHandler.updateCompactorState(ci, 0);
    txnHandler.markCompacted(ci);
    toClean = txnHandler.findReadyToClean(0, 0);
    assertEquals(1, toClean.size());
    txnHandler.markCleaned(ci);
    txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
    // The open txn will became the low water mark
    Thread.sleep(txnHandler.getOpenTxnTimeOutMillis());
    txnHandler.setOpenTxnTimeOutMillis(1);
    txnHandler.cleanEmptyAbortedAndCommittedTxns();
    txnList = txnHandler.getOpenTxns();
    assertEquals(3, txnList.getOpen_txnsSize());
    txnHandler.setOpenTxnTimeOutMillis(1000);
}

Example 17 with OpenTxnRequest

use of org.apache.hadoop.hive.metastore.api.OpenTxnRequest in project hive by apache.

the class TestTxnHandler method testValidTxnsNoneOpen.

@Test
public void testValidTxnsNoneOpen() throws Exception {
    txnHandler.openTxns(new OpenTxnRequest(2, "me", "localhost"));
    txnHandler.commitTxn(new CommitTxnRequest(1));
    txnHandler.commitTxn(new CommitTxnRequest(2));
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(2L, txnsInfo.getTxn_high_water_mark());
    assertEquals(0, txnsInfo.getOpen_txns().size());
    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(2L, txns.getTxn_high_water_mark());
    assertEquals(0, txns.getOpen_txns().size());
}

Example 18 with OpenTxnRequest

use of org.apache.hadoop.hive.metastore.api.OpenTxnRequest in project hive by apache.

the class TestTxnHandler method testAbortTxns.

@Test
public void testAbortTxns() throws Exception {
    createDatabaseForReplTests("default", MetaStoreUtils.getDefaultCatalog(conf));
    OpenTxnsResponse openedTxns = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    List<Long> txnList = openedTxns.getTxn_ids();
    txnHandler.abortTxns(new AbortTxnsRequest(txnList));
    OpenTxnRequest replRqst = new OpenTxnRequest(2, "me", "localhost");
    replRqst.setReplPolicy("default.*");
    replRqst.setTxn_type(TxnType.REPL_CREATED);
    replRqst.setReplSrcTxnIds(Arrays.asList(1L, 2L));
    List<Long> targetTxns = txnHandler.openTxns(replRqst).getTxn_ids();
    assertTrue(targetTxnsPresentInReplTxnMap(1L, 2L, targetTxns));
    txnHandler.abortTxns(new AbortTxnsRequest(targetTxns));
    assertFalse(targetTxnsPresentInReplTxnMap(1L, 2L, targetTxns));
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(5, txnsInfo.getOpen_txns().size());
    txnsInfo.getOpen_txns().forEach(txn -> assertEquals(TxnState.ABORTED, txn.getState()));
}

Example 19 with OpenTxnRequest

use of org.apache.hadoop.hive.metastore.api.OpenTxnRequest in project hive by apache.

the class TestTxnHandler method testValidTxnsSomeOpen.

@Test
public void testValidTxnsSomeOpen() throws Exception {
    txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    txnHandler.abortTxn(new AbortTxnRequest(1));
    txnHandler.commitTxn(new CommitTxnRequest(2));
    GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo();
    assertEquals(3L, txnsInfo.getTxn_high_water_mark());
    assertEquals(2, txnsInfo.getOpen_txns().size());
    assertEquals(1L, txnsInfo.getOpen_txns().get(0).getId());
    assertEquals(TxnState.ABORTED, txnsInfo.getOpen_txns().get(0).getState());
    assertEquals(3L, txnsInfo.getOpen_txns().get(1).getId());
    assertEquals(TxnState.OPEN, txnsInfo.getOpen_txns().get(1).getState());
    GetOpenTxnsResponse txns = txnHandler.getOpenTxns();
    assertEquals(3L, txns.getTxn_high_water_mark());
    assertEquals(2, txns.getOpen_txns().size());
    boolean[] saw = new boolean[4];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (Long tid : txns.getOpen_txns()) {
        saw[tid.intValue()] = true;
    }
    assertTrue(saw[1]);
    assertFalse(saw[2]);
    assertTrue(saw[3]);
}

Example 20 with OpenTxnRequest

use of org.apache.hadoop.hive.metastore.api.OpenTxnRequest in project hive by apache.

the class TestTxnCommands2 method testOpenTxnsCounter.

@Test
public void testOpenTxnsCounter() throws Exception {
    hiveConf.setIntVar(HiveConf.ConfVars.HIVE_MAX_OPEN_TXNS, 3);
    hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_COUNT_OPEN_TXNS_INTERVAL, 10, TimeUnit.MILLISECONDS);
    OpenTxnsResponse openTxnsResponse = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
    AcidOpenTxnsCounterService openTxnsCounterService = new AcidOpenTxnsCounterService();
    openTxnsCounterService.setConf(hiveConf);
    // will update current number of open txns to 3
    openTxnsCounterService.run();
    MetaException exception = null;
    // This should fail once it finds out the threshold has been reached
    try {
        txnHandler.openTxns(new OpenTxnRequest(1, "you", "localhost"));
    } catch (MetaException e) {
        exception = e;
    }
    Assert.assertNotNull("Opening new transaction shouldn't be allowed", exception);
    Assert.assertTrue(exception.getMessage().equals("Maximum allowed number of open transactions has been reached. See hive.max.open.txns."));
    // new transactions should be allowed to open
    for (long txnid : openTxnsResponse.getTxn_ids()) {
        txnHandler.commitTxn(new CommitTxnRequest(txnid));
    }
    // will update current number of open txns back to 0
    openTxnsCounterService.run();
    exception = null;
    try {
        txnHandler.openTxns(new OpenTxnRequest(1, "him", "localhost"));
    } catch (MetaException e) {
        exception = e;
    }
    Assert.assertNull(exception);
}