Examples with ShowCompactResponseElement org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement used on opensource projects

Example 91 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCrudCompactorOnTez method testStatsAfterQueryCompactionOnTez.

/**
 * After each major compaction, stats need to be updated on the table
 * 1. create an ORC backed table (Orc is currently required by ACID)
 * 2. populate with data
 * 3. compute stats
 * 4. Trigger major compaction (which should update stats)
 * 5. check that stats have been updated
 */
@Test
public void testStatsAfterQueryCompactionOnTez() throws Exception {
    // as of (8/27/2014) Hive 0.14, ACID/Orc requires HiveInputFormat
    String dbName = "default";
    String tblName = "compaction_test";
    executeStatementOnDriver("drop table if exists " + tblName, driver);
    executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + // currently ACID requires table to be bucketed
    " CLUSTERED BY(a) INTO 4 BUCKETS" + " STORED AS ORC  TBLPROPERTIES ('transactional'='true')", driver);
    executeStatementOnDriver("INSERT INTO TABLE " + tblName + " values(55, 'London')", driver);
    executeStatementOnDriver("INSERT INTO TABLE " + tblName + " values(56, 'Paris')", driver);
    execSelectAndDumpData("select * from " + tblName, driver, "Dumping data for " + tblName + " after load:");
    TxnStore txnHandler = TxnUtils.getTxnStore(conf);
    Table table = msClient.getTable(dbName, tblName);
    // compute stats before compaction
    CompactionInfo ci = new CompactionInfo(dbName, tblName, null, CompactionType.MAJOR);
    Worker.StatsUpdater.gatherStats(ci, conf, System.getProperty("user.name"), CompactorUtil.getCompactorJobQueueName(conf, ci, table));
    // Check basic stats are collected
    Map<String, String> parameters = Hive.get().getTable(tblName).getParameters();
    Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles"));
    Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows"));
    Assert.assertEquals("The total table size is differing from the expected", "1434", parameters.get("totalSize"));
    // Do a major compaction
    CompactorTestUtil.runCompaction(conf, dbName, tblName, CompactionType.MAJOR, true);
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    if (1 != compacts.size()) {
        Assert.fail("Expecting 1 file and found " + compacts.size() + " files " + compacts);
    }
    Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
    // Check basic stats are updated
    parameters = Hive.get().getTable(tblName).getParameters();
    Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles"));
    Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows"));
    Assert.assertEquals("The total table size is differing from the expected", "727", parameters.get("totalSize"));
}

Example 92 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCrudCompactorOnTez method testMinorCompactionWithoutBucketsCommon.

private void testMinorCompactionWithoutBucketsCommon(String dbName, String tableName, String tempTableName, boolean insertOverWrite, List<String> expectedDeltas, List<String> expectedDeleteDeltas, String expectedCompactedDeltaDirName, CompactionType compactionType) throws Exception {
    TestDataProvider dataProvider = new TestDataProvider();
    dataProvider.createTableWithoutBucketWithMultipleSplits(dbName, tableName, tempTableName, true, true, insertOverWrite);
    FileSystem fs = FileSystem.get(conf);
    Table table = msClient.getTable(dbName, tableName);
    List<String> expectedData = dataProvider.getAllData(tableName);
    List<String> expectedFileNames = dataProvider.getDataWithInputFileNames(null, tableName);
    // Verify deltas
    Assert.assertEquals("Delta directories does not match", expectedDeltas, CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null));
    // Verify delete delta
    Assert.assertEquals("Delete directories does not match", expectedDeleteDeltas, CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deleteEventDeltaDirFilter, table, null));
    List<String> expectedBucketFiles = CompactorTestUtil.getBucketFileNamesWithoutAttemptId(fs, table, null, expectedDeltas);
    List<String> expectedDeleteBucketFiles = CompactorTestUtil.getBucketFileNamesWithoutAttemptId(fs, table, null, expectedDeleteDeltas);
    CompactorTestUtil.runCompaction(conf, dbName, tableName, compactionType, true);
    // Clean up resources
    CompactorTestUtil.runCleaner(conf);
    // Only 1 compaction should be in the response queue with succeeded state
    List<ShowCompactResponseElement> compacts = TxnUtils.getTxnStore(conf).showCompact(new ShowCompactRequest()).getCompacts();
    Assert.assertEquals("Completed compaction queue must contain one element", 1, compacts.size());
    Assert.assertEquals("Compaction state is not succeeded", "succeeded", compacts.get(0).getState());
    // Verify delta and delete delta directories after compaction
    if (CompactionType.MAJOR == compactionType) {
        List<String> actualBasesAfterComp = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null);
        Assert.assertEquals("Base directory does not match after compaction", Collections.singletonList(expectedCompactedDeltaDirName), actualBasesAfterComp);
        // Verify bucket files in delta and delete delta dirs
        Assert.assertEquals("Bucket names are not matching after compaction in the base folder", expectedBucketFiles, CompactorTestUtil.getBucketFileNames(fs, table, null, actualBasesAfterComp.get(0)));
    } else {
        List<String> actualDeltasAfterComp = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
        Assert.assertEquals("Delta directories does not match after compaction", Collections.singletonList(expectedCompactedDeltaDirName), actualDeltasAfterComp);
        List<String> actualDeleteDeltasAfterComp = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deleteEventDeltaDirFilter, table, null);
        Assert.assertEquals("Delete delta directories does not match after compaction", Collections.singletonList("delete_" + expectedCompactedDeltaDirName), actualDeleteDeltasAfterComp);
        // Verify bucket files in delta and delete delta dirs
        Assert.assertEquals("Bucket names are not matching after compaction in the delete deltas", expectedDeleteBucketFiles, CompactorTestUtil.getBucketFileNames(fs, table, null, actualDeleteDeltasAfterComp.get(0)));
        Assert.assertEquals("Bucket names are not matching after compaction", expectedBucketFiles, CompactorTestUtil.getBucketFileNames(fs, table, null, actualDeltasAfterComp.get(0)));
    }
    // Verify all contents
    List<String> actualData = dataProvider.getAllData(tableName);
    Assert.assertEquals(expectedData, actualData);
    List<String> actualFileNames = dataProvider.getDataWithInputFileNames(null, tableName);
    Assert.assertTrue(dataProvider.compareFileNames(expectedFileNames, actualFileNames));
    dataProvider.dropTable(tableName);
}

Example 93 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestMmCompactorOnTez method testMmMinorCompaction10DeltaDirs.

@Test
public void testMmMinorCompaction10DeltaDirs() throws Exception {
    String dbName = "default";
    String tableName = "testMmMinorCompaction";
    // Create test table
    TestDataProvider dataProvider = new TestDataProvider();
    dataProvider.createMmTable(tableName, false, false);
    // Find the location of the table
    IMetaStoreClient metaStoreClient = new HiveMetaStoreClient(conf);
    Table table = metaStoreClient.getTable(dbName, tableName);
    FileSystem fs = FileSystem.get(conf);
    // Insert test data into test table
    dataProvider.insertMmTestData(tableName, 10);
    // Get all data before compaction is run
    List<String> expectedData = dataProvider.getAllData(tableName);
    Collections.sort(expectedData);
    // Verify deltas
    List<String> deltaNames = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
    Assert.assertEquals(10, deltaNames.size());
    // Run a compaction
    CompactorTestUtil.runCompaction(conf, dbName, tableName, CompactionType.MINOR, true);
    CompactorTestUtil.runCleaner(conf);
    List<ShowCompactResponseElement> compacts = TxnUtils.getTxnStore(conf).showCompact(new ShowCompactRequest()).getCompacts();
    Assert.assertEquals("Completed compaction queue must contain 3 element", 1, compacts.size());
    compacts.forEach(c -> Assert.assertEquals("Compaction state is not succeeded", "succeeded", c.getState()));
    // Verify delta directories after compaction
    List<String> actualDeltasAfterComp = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
    Assert.assertEquals(Collections.singletonList("delta_0000001_0000010_v0000014"), actualDeltasAfterComp);
    // Verify bucket file in delta dir
    List<String> expectedBucketFile = Collections.singletonList("000000_0");
    Assert.assertEquals("Bucket names are not matching after compaction", expectedBucketFile, CompactorTestUtil.getBucketFileNamesForMMTables(fs, table, null, actualDeltasAfterComp.get(0)));
    verifyAllContents(tableName, dataProvider, expectedData);
    // Clean up
    dataProvider.dropTable(tableName);
}

Example 94 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetricData method aCompaction.

private static ShowCompactResponseElement aCompaction(long id, String table, String partition, String state, CompactionType type, Long enqueuedTime, Long startTime, Long cleanerStart) {
    ShowCompactResponseElement e = new ShowCompactResponseElement("db_name", table, type, state);
    e.setId(id);
    e.setPartitionname(partition);
    if (enqueuedTime != null) {
        e.setEnqueueTime(enqueuedTime);
    }
    if (startTime != null) {
        e.setStart(startTime);
    }
    if (cleanerStart != null) {
        e.setCleanerStart(cleanerStart);
    }
    return e;
}

Example 95 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetricData method aCompaction.

private static ShowCompactResponseElement aCompaction(long id, String table, String partition, String state, CompactionType type) {
    ShowCompactResponseElement e = new ShowCompactResponseElement("db_name", table, type, state);
    e.setId(id);
    e.setPartitionname(partition);
    return e;
}