Examples with TxnStore org.apache.hadoop.hive.metastore.txn.TxnStore used on opensource projects

Example 61 with TxnStore

use of org.apache.hadoop.hive.metastore.txn.TxnStore in project hive by apache.

the class TestTxnCommands method testVersioning.

@Test
public void testVersioning() throws Exception {
    hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true");
    runStatementOnDriver("drop table if exists T");
    runStatementOnDriver("create table T (a int, b int) stored as orc");
    int[][] data = { { 1, 2 } };
    // create 1 delta file bucket_00000
    runStatementOnDriver("insert into T" + makeValuesClause(data));
    // delete the bucket files so now we have empty delta dirs
    List<String> rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T");
    FileSystem fs = FileSystem.get(hiveConf);
    Assert.assertTrue(rs != null && rs.size() == 1 && rs.get(0).contains(AcidUtils.DELTA_PREFIX));
    Path filePath = new Path(rs.get(0));
    int version = AcidUtils.OrcAcidVersion.getAcidVersionFromDataFile(filePath, fs);
    // check it has expected version marker
    Assert.assertEquals("Unexpected version marker in " + filePath, AcidUtils.OrcAcidVersion.ORC_ACID_VERSION, version);
    // check that delta dir has a version file with expected value
    filePath = filePath.getParent();
    Assert.assertTrue(filePath.getName().startsWith(AcidUtils.DELTA_PREFIX));
    int versionFromMetaFile = AcidUtils.OrcAcidVersion.getAcidVersionFromMetaFile(filePath, fs);
    Assert.assertEquals("Unexpected version marker in " + filePath, AcidUtils.OrcAcidVersion.ORC_ACID_VERSION, versionFromMetaFile);
    runStatementOnDriver("insert into T" + makeValuesClause(data));
    runStatementOnDriver("alter table T compact 'major'");
    TestTxnCommands2.runWorker(hiveConf);
    // check status of compaction job
    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
    Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
    Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local"));
    rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T");
    Assert.assertTrue(rs != null && rs.size() == 1 && rs.get(0).contains(AcidUtils.BASE_PREFIX));
    filePath = new Path(rs.get(0));
    version = AcidUtils.OrcAcidVersion.getAcidVersionFromDataFile(filePath, fs);
    // check that files produced by compaction still have the version marker
    Assert.assertEquals("Unexpected version marker in " + filePath, AcidUtils.OrcAcidVersion.ORC_ACID_VERSION, version);
    // check that compacted base dir has a version file with expected value
    filePath = filePath.getParent();
    Assert.assertTrue(filePath.getName().startsWith(AcidUtils.BASE_PREFIX));
    versionFromMetaFile = AcidUtils.OrcAcidVersion.getAcidVersionFromMetaFile(filePath, fs);
    Assert.assertEquals("Unexpected version marker in " + filePath, AcidUtils.OrcAcidVersion.ORC_ACID_VERSION, versionFromMetaFile);
}