Examples with TxnStore org.apache.hadoop.hive.metastore.txn.TxnStore used on opensource projects

Example 56 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TestAcidOnTez method testJoin.

// Ideally test like this should be a qfile test. However, the explain output from qfile is always
// slightly different depending on where the test is run, specifically due to file size estimation
private void testJoin(String engine, String joinType) throws Exception {
    // make a clone of existing hive conf
    HiveConf confForTez = new HiveConf(hiveConf);
    // make a clone of existing hive conf
    HiveConf confForMR = new HiveConf(hiveConf);
    if (engine.equals("tez")) {
        // one-time setup to make query able to run with Tez
        setupTez(confForTez);
    }
    if (joinType.equals("MapJoin")) {
        setupMapJoin(confForTez);
        setupMapJoin(confForMR);
    }
    runQueries(engine, joinType, confForTez, confForMR);
    // Perform compaction. Join result after compaction should still be the same
    runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'");
    runWorker(hiveConf);
    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
    Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
    runCleaner(hiveConf);
    runQueries(engine, joinType, confForTez, confForMR);
}

Example 57 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TestAcidOnTez method testGetSplitsLocksWithMaterializedView.

@Test
public void testGetSplitsLocksWithMaterializedView() throws Exception {
    // Need to test this with LLAP settings, which requires some additional configurations set.
    HiveConf modConf = new HiveConf(hiveConf);
    setupTez(modConf);
    modConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
    modConf.setVar(ConfVars.HIVEFETCHTASKCONVERSION, "more");
    modConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost");
    // SessionState/Driver needs to be restarted with the Tez conf settings.
    restartSessionAndDriver(modConf);
    TxnStore txnHandler = TxnUtils.getTxnStore(modConf);
    String mvName = "mv_acidTbl";
    try {
        runStatementOnDriver("create materialized view " + mvName + " as select a from " + Table.ACIDTBL + " where a > 5");
        // Request LLAP splits for a table.
        String queryParam = "select a from " + Table.ACIDTBL + " where a > 5";
        runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
        // The get_splits call should have resulted in a lock on ACIDTBL and materialized view mv_acidTbl
        ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
        TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks());
        TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", mvName, null, slr.getLocks());
        assertEquals(2, slr.getLocksSize());
    } finally {
        // Close the session which should free up the TxnHandler/locks held by the session.
        // Done in the finally block to make sure we free up the locks; otherwise
        // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL.
        restartSessionAndDriver(hiveConf);
        runStatementOnDriver("drop materialized view if exists " + mvName);
    }
    // Lock should be freed up now.
    ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
    assertEquals(0, slr.getLocksSize());
    List<String> rows = runStatementOnDriver("show transactions");
    // Transactions should be committed.
    // No transactions - just the header row
    assertEquals(1, rows.size());
}

Example 58 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TestAcidOnTez method testGetSplitsLocks.

@Test
public void testGetSplitsLocks() throws Exception {
    // Need to test this with LLAP settings, which requires some additional configurations set.
    HiveConf modConf = new HiveConf(hiveConf);
    setupTez(modConf);
    modConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
    modConf.setVar(ConfVars.HIVEFETCHTASKCONVERSION, "more");
    modConf.setVar(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "localhost");
    // SessionState/Driver needs to be restarted with the Tez conf settings.
    restartSessionAndDriver(modConf);
    TxnStore txnHandler = TxnUtils.getTxnStore(modConf);
    try {
        // Request LLAP splits for a table.
        String queryParam = "select * from " + Table.ACIDTBL;
        runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
        // The get_splits call should have resulted in a lock on ACIDTBL
        ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
        TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks());
        assertEquals(1, slr.getLocksSize());
        // Try another table.
        queryParam = "select * from " + Table.ACIDTBLPART;
        runStatementOnDriver("select get_splits(\"" + queryParam + "\", 1)");
        // Should now have new lock on ACIDTBLPART
        slr = txnHandler.showLocks(new ShowLocksRequest());
        TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBLPART.name, null, slr.getLocks());
        assertEquals(2, slr.getLocksSize());
        // There should be different txn IDs associated with each lock.
        Set<Long> txnSet = new HashSet<Long>();
        for (ShowLocksResponseElement lockResponseElem : slr.getLocks()) {
            txnSet.add(lockResponseElem.getTxnid());
        }
        assertEquals(2, txnSet.size());
        List<String> rows = runStatementOnDriver("show transactions");
        // Header row + 2 transactions = 3 rows
        assertEquals(3, rows.size());
    } finally {
        // Close the session which should free up the TxnHandler/locks held by the session.
        // Done in the finally block to make sure we free up the locks; otherwise
        // the cleanup in tearDown() will get stuck waiting on the lock held here on ACIDTBL.
        restartSessionAndDriver(hiveConf);
    }
    // Lock should be freed up now.
    ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest());
    assertEquals(0, slr.getLocksSize());
    List<String> rows = runStatementOnDriver("show transactions");
    // Transactions should be committed.
    // No transactions - just the header row
    assertEquals(1, rows.size());
}

Example 59 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TransactionalValidationListener method handleAlterTableTransactionalProp.

/**
 * once a table is marked transactional, you cannot go back.  Enforce this.
 * Also in current version, 'transactional_properties' of the table cannot be altered after
 * the table is created. Any attempt to alter it will throw a MetaException.
 */
private void handleAlterTableTransactionalProp(PreAlterTableEvent context) throws MetaException {
    Table newTable = context.getNewTable();
    Map<String, String> parameters = newTable.getParameters();
    if (parameters == null || parameters.isEmpty()) {
        return;
    }
    Set<String> keys = new HashSet<>(parameters.keySet());
    String transactionalValue = null;
    boolean transactionalValuePresent = false;
    boolean isTransactionalPropertiesPresent = false;
    String transactionalPropertiesValue = null;
    boolean hasValidTransactionalValue = false;
    for (String key : keys) {
        if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
            transactionalValuePresent = true;
            transactionalValue = parameters.get(key);
            parameters.remove(key);
        }
        if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) {
            isTransactionalPropertiesPresent = true;
            transactionalPropertiesValue = parameters.get(key);
        // Do not remove the parameter yet, because we have separate initialization routine
        // that will use it down below.
        }
    }
    Table oldTable = context.getOldTable();
    String oldTransactionalValue = null;
    String oldTransactionalPropertiesValue = null;
    for (String key : oldTable.getParameters().keySet()) {
        if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
            oldTransactionalValue = oldTable.getParameters().get(key);
        }
        if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) {
            oldTransactionalPropertiesValue = oldTable.getParameters().get(key);
        }
    }
    if (transactionalValuePresent && "false".equalsIgnoreCase(transactionalValue)) {
        transactionalValuePresent = false;
        transactionalValue = null;
    }
    if (transactionalValuePresent) {
        if (oldTable.getTableType().equals(TableType.MANAGED_TABLE.toString()) && newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
            throw new MetaException(Warehouse.getQualifiedName(newTable) + " cannot be converted to external table as it is transactional table.");
        }
        // normalize prop name
        parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue);
    }
    if ("true".equalsIgnoreCase(transactionalValue) && !"true".equalsIgnoreCase(oldTransactionalValue)) {
        if (!isTransactionalPropertiesPresent) {
            normalizeTransactionalPropertyDefault(newTable);
            isTransactionalPropertiesPresent = true;
            transactionalPropertiesValue = DEFAULT_TRANSACTIONAL_PROPERTY;
        }
        // We only need to check conformance if alter table enabled acid.
        // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing.
        boolean isFullAcid = transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue);
        if (isFullAcid && !conformToAcid(newTable)) {
            throw new MetaException("The table must be stored using an ACID compliant " + "format (such as ORC): " + Warehouse.getQualifiedName(newTable));
        }
        if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
            throw new MetaException(Warehouse.getQualifiedName(newTable) + " cannot be declared transactional because it's an external table");
        }
        if (isFullAcid) {
            validateTableStructure(context.getHandler(), newTable);
        }
        hasValidTransactionalValue = true;
    }
    if (oldTransactionalValue == null ? transactionalValue == null : oldTransactionalValue.equalsIgnoreCase(transactionalValue)) {
        // this covers backward compat cases where this prop may have been set already
        hasValidTransactionalValue = true;
    }
    if (!hasValidTransactionalValue && !MetaStoreUtils.isInsertOnlyTableParam(oldTable.getParameters())) {
        // and NOT the same value it was before
        throw new MetaException("TBLPROPERTIES with 'transactional'='true' cannot be unset: " + Warehouse.getQualifiedName(newTable));
    }
    if (isTransactionalPropertiesPresent) {
        // Now validate transactional_properties for the table.
        if (oldTransactionalValue == null) {
            // If this is the first time the table is being initialized to 'transactional=true',
            // any valid value can be set for the 'transactional_properties'.
            initializeTransactionalProperties(newTable);
        } else {
            // null and an attempt is made to set it. This behaviour can be changed in the future.
            if ((oldTransactionalPropertiesValue == null || !oldTransactionalPropertiesValue.equalsIgnoreCase(transactionalPropertiesValue)) && !MetaStoreUtils.isInsertOnlyTableParam(oldTable.getParameters())) {
                throw new MetaException("TBLPROPERTIES with 'transactional_properties' cannot be " + "altered after the table is created");
            }
        }
    }
    checkSorted(newTable);
    if (TxnUtils.isAcidTable(newTable) && !TxnUtils.isAcidTable(oldTable)) {
        /* we just made an existing table full acid which wasn't acid before and it passed all checks
      initialize the Write ID sequence so that we can handle assigning ROW_IDs to 'original'
      files already present in the table. */
        TxnStore t = TxnUtils.getTxnStore(getConf());
        // For now assume no partition may have > 10M files.  Perhaps better to count them.
        t.seedWriteId(new SeedTableWriteIdsRequest(newTable.getDbName(), newTable.getTableName(), 10000000));
    }
}

Example 60 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TestPreUpgradeTool method testUpgrade.

/**
 * preUpgrade: test tables that need to be compacted, waits for compaction
 * postUpgrade: generates scripts w/o asserts
 */
@Test
public void testUpgrade() throws Exception {
    int[][] data = { { 1, 2 }, { 3, 4 }, { 5, 6 } };
    int[][] dataPart = { { 1, 2, 10 }, { 3, 4, 11 }, { 5, 6, 12 } };
    runStatementOnDriver("drop table if exists TAcid");
    runStatementOnDriver("drop table if exists TAcidPart");
    runStatementOnDriver("drop table if exists TFlat");
    runStatementOnDriver("drop table if exists TFlatText");
    try {
        runStatementOnDriver("create table TAcid (a int, b int) clustered by (b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
        runStatementOnDriver("create table TAcidPart (a int, b int) partitioned by (p tinyint)  clustered by (b) into 2 buckets  stored" + " as orc TBLPROPERTIES ('transactional'='true')");
        // on 2.x these are guaranteed to not be acid
        runStatementOnDriver("create table TFlat (a int, b int) stored as orc tblproperties('transactional'='false')");
        runStatementOnDriver("create table TFlatText (a int, b int) stored as textfile tblproperties('transactional'='false')");
        // this needs major compaction
        runStatementOnDriver("insert into TAcid" + makeValuesClause(data));
        runStatementOnDriver("update TAcid set a = 1 where b = 2");
        // this table needs to be converted to CRUD Acid
        runStatementOnDriver("insert into TFlat" + makeValuesClause(data));
        // this table needs to be converted to MM
        runStatementOnDriver("insert into TFlatText" + makeValuesClause(data));
        // p=10 needs major compaction
        runStatementOnDriver("insert into TAcidPart partition(p)" + makeValuesClause(dataPart));
        runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p = 10");
        // todo: add partitioned table that needs conversion to MM/Acid
        // todo: rename files case
        String[] args = { "-location", getTestDataDir(), "-execute" };
        PreUpgradeTool.callback = new PreUpgradeTool.Callback() {

            @Override
            void onWaitForCompaction() throws MetaException {
                runWorker(hiveConf);
            }
        };
        PreUpgradeTool.pollIntervalMs = 1;
        PreUpgradeTool.hiveConf = hiveConf;
        PreUpgradeTool.main(args);
        String[] scriptFiles = getScriptFiles();
        assertThat(scriptFiles.length, is(1));
        List<String> scriptContent = loadScriptContent(new File(getTestDataDir(), scriptFiles[0]));
        assertThat(scriptContent.size(), is(2));
        assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacid COMPACT 'major';")));
        assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';")));
        TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
        ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
        Assert.assertEquals(2, resp.getCompactsSize());
        for (ShowCompactResponseElement e : resp.getCompacts()) {
            Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState());
        }
        // Check whether compaction was successful in the first run
        File secondRunDataDir = new File(getTestDataDir(), "secondRun");
        if (!secondRunDataDir.exists()) {
            if (!secondRunDataDir.mkdir()) {
                throw new IOException("Unable to create directory" + secondRunDataDir.getAbsolutePath());
            }
        }
        String[] args2 = { "-location", secondRunDataDir.getAbsolutePath() };
        PreUpgradeTool.main(args2);
        scriptFiles = secondRunDataDir.list();
        assertThat(scriptFiles, is(not(nullValue())));
        assertThat(scriptFiles.length, is(0));
    } finally {
        runStatementOnDriver("drop table if exists TAcid");
        runStatementOnDriver("drop table if exists TAcidPart");
        runStatementOnDriver("drop table if exists TFlat");
        runStatementOnDriver("drop table if exists TFlatText");
    }
}