Examples with GetOpenTxnsResponse org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse used on opensource projects

Example 6 with GetOpenTxnsResponse

use of org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse in project hive by apache.

the class TestTxnCommands3 method testMajorCompactionAbortLeftoverFiles.

@Test
public void testMajorCompactionAbortLeftoverFiles() throws Exception {
    MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true);
    dropTable(new String[] { "T" });
    // note: transaction names T1, T2, etc below, are logical, the actual txnid will be different
    runStatementOnDriver("create table T (a int, b int) stored as orc");
    // makes delta_1_1 in T1
    runStatementOnDriver("insert into T values(0,2)");
    // makes delta_2_2 in T2
    runStatementOnDriver("insert into T values(1,4)");
    runStatementOnDriver("alter table T compact 'minor'");
    // create failed compaction attempt so that compactor txn is aborted
    CompactorMR compactorMr = Mockito.spy(new CompactorMR());
    Mockito.doAnswer((Answer<Void>) invocationOnMock -> {
        invocationOnMock.callRealMethod();
        throw new RuntimeException("Will cause CompactorMR to fail all opening txn and creating directories for compaction.");
    }).when(compactorMr).run(any(), any(), any(), any(), any(), any(), any(), any(), any());
    Worker worker = Mockito.spy(new Worker());
    worker.setConf(hiveConf);
    worker.init(new AtomicBoolean(true));
    Mockito.doReturn(compactorMr).when(worker).getMrCompactor();
    worker.run();
    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
    Assert.assertEquals("Unexpected 0th compaction state", TxnStore.FAILED_RESPONSE, resp.getCompacts().get(0).getState());
    GetOpenTxnsResponse openResp = txnHandler.getOpenTxns();
    Assert.assertEquals(openResp.toString(), 1, openResp.getOpen_txnsSize());
    // check that the compactor txn is aborted
    Assert.assertTrue(openResp.toString(), BitSet.valueOf(openResp.getAbortedBits()).get(0));
    Assert.assertEquals(0, TestTxnDbUtil.countQueryAgent(hiveConf, "SELECT count(*) FROM hive_locks WHERE hl_txnid=" + openResp.getOpen_txns().get(0)));
    FileSystem fs = FileSystem.get(hiveConf);
    Path warehousePath = new Path(getWarehouseDir());
    FileStatus[] actualList = fs.listStatus(new Path(warehousePath + "/t"), FileUtils.HIDDEN_FILES_PATH_FILTER);
    // we expect all the t/base_* files to be removed by the compactor failure
    String[] expectedList = new String[] { "/t/delta_0000001_0000001_0000", "/t/delta_0000002_0000002_0000" };
    checkExpectedFiles(actualList, expectedList, warehousePath.toString());
    // delete metadata about aborted txn from txn_components and files (if any)
    runCleaner(hiveConf);
}

Example 7 with GetOpenTxnsResponse

use of org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse in project hive by apache.

the class TestOpenTxn method testMultiGapWithOldCommit.

@Test
public void testMultiGapWithOldCommit() throws Exception {
    OpenTxnRequest openTxnRequest = new OpenTxnRequest(1, "me", "localhost");
    long first = txnHandler.openTxns(openTxnRequest).getTxn_ids().get(0);
    txnHandler.commitTxn(new CommitTxnRequest(first));
    long second = txnHandler.openTxns(new OpenTxnRequest(10, "me", "localhost")).getTxn_ids().get(0);
    deleteTransaction(second, second + 9);
    txnHandler.openTxns(openTxnRequest);
    GetOpenTxnsResponse openTxns = txnHandler.getOpenTxns();
    Assert.assertEquals(11, openTxns.getOpen_txnsSize());
}

Example 8 with GetOpenTxnsResponse

use of org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse in project hive by apache.

the class TestOpenTxn method testGapWithOldCommit.

@Test
public void testGapWithOldCommit() throws Exception {
    OpenTxnRequest openTxnRequest = new OpenTxnRequest(1, "me", "localhost");
    long first = txnHandler.openTxns(openTxnRequest).getTxn_ids().get(0);
    txnHandler.commitTxn(new CommitTxnRequest(first));
    long second = txnHandler.openTxns(openTxnRequest).getTxn_ids().get(0);
    deleteTransaction(second);
    txnHandler.openTxns(openTxnRequest);
    GetOpenTxnsResponse openTxns = txnHandler.getOpenTxns();
    Assert.assertEquals(2, openTxns.getOpen_txnsSize());
}

Example 9 with GetOpenTxnsResponse

use of org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse in project hive by apache.

the class HMSClient method getOpenTxns.

List<Long> getOpenTxns() throws TException {
    GetOpenTxnsRequest getOpenTxnsRequest = new GetOpenTxnsRequest();
    getOpenTxnsRequest.setExcludeTxnTypes(Arrays.asList(TxnType.READ_ONLY));
    GetOpenTxnsResponse txns = client.get_open_txns_req(getOpenTxnsRequest);
    List<Long> openTxns = new ArrayList<>();
    BitSet abortedBits = BitSet.valueOf(txns.getAbortedBits());
    int i = 0;
    for (long txnId : txns.getOpen_txns()) {
        if (!abortedBits.get(i)) {
            openTxns.add(txnId);
        }
        ++i;
    }
    return openTxns;
}

Example 10 with GetOpenTxnsResponse

use of org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse in project hive by apache.

the class TxnHandler method getOpenTxns.

@Override
@RetrySemantics.ReadOnly
public GetOpenTxnsResponse getOpenTxns() throws MetaException {
    try {
        // We need to figure out the current transaction number and the list of
        // open transactions.  To avoid needing a transaction on the underlying
        // database we'll look at the current transaction number first.  If it
        // subsequently shows up in the open list that's ok.
        Connection dbConn = null;
        Statement stmt = null;
        ResultSet rs = null;
        try {
            /**
             * This runs at READ_COMMITTED for exactly the same reason as {@link #getOpenTxnsInfo()}
             */
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            String s = "select ntxn_next - 1 from NEXT_TXN_ID";
            LOG.debug("Going to execute query <" + s + ">");
            rs = stmt.executeQuery(s);
            if (!rs.next()) {
                throw new MetaException("Transaction tables not properly " + "initialized, no record found in next_txn_id");
            }
            long hwm = rs.getLong(1);
            if (rs.wasNull()) {
                throw new MetaException("Transaction tables not properly " + "initialized, null record found in next_txn_id");
            }
            close(rs);
            List<Long> openList = new ArrayList<>();
            // need the WHERE clause below to ensure consistent results with READ_COMMITTED
            s = "select txn_id, txn_state from TXNS where txn_id <= " + hwm + " order by txn_id";
            LOG.debug("Going to execute query<" + s + ">");
            rs = stmt.executeQuery(s);
            long minOpenTxn = Long.MAX_VALUE;
            BitSet abortedBits = new BitSet();
            while (rs.next()) {
                long txnId = rs.getLong(1);
                openList.add(txnId);
                char c = rs.getString(2).charAt(0);
                if (c == TXN_OPEN) {
                    minOpenTxn = Math.min(minOpenTxn, txnId);
                } else if (c == TXN_ABORTED) {
                    abortedBits.set(openList.size() - 1);
                }
            }
            LOG.debug("Going to rollback");
            dbConn.rollback();
            ByteBuffer byteBuffer = ByteBuffer.wrap(abortedBits.toByteArray());
            GetOpenTxnsResponse otr = new GetOpenTxnsResponse(hwm, openList, byteBuffer);
            if (minOpenTxn < Long.MAX_VALUE) {
                otr.setMin_open_txn(minOpenTxn);
            }
            return otr;
        } catch (SQLException e) {
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "getOpenTxns");
            throw new MetaException("Unable to select from transaction database, " + StringUtils.stringifyException(e));
        } finally {
            close(rs, stmt, dbConn);
        }
    } catch (RetryException e) {
        return getOpenTxns();
    }
}