Search in sources :

Example 16 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestTxnCommands2 method testSchemaEvolutionCompaction.

/**
 * Create a table with schema evolution, and verify that no data is lost during (MR major)
 * compaction.
 *
 * @throws Exception if a query fails
 */
@Test
public void testSchemaEvolutionCompaction() throws Exception {
    String tblName = "schemaevolutioncompaction";
    runStatementOnDriver("drop table if exists " + tblName);
    runStatementOnDriver("CREATE TABLE " + tblName + "(a INT) " + " PARTITIONED BY(part string)" + " STORED AS ORC TBLPROPERTIES ('transactional'='true')");
    // First INSERT round.
    runStatementOnDriver("insert into " + tblName + " partition (part='aa') values (1)");
    // ALTER TABLE ... ADD COLUMNS
    runStatementOnDriver("ALTER TABLE " + tblName + " ADD COLUMNS(b int)");
    // Second INSERT round.
    runStatementOnDriver("insert into " + tblName + " partition (part='aa') values (2, 2000)");
    // Validate data
    List<String> res = runStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a");
    Assert.assertEquals(2, res.size());
    Assert.assertEquals("1\tNULL\taa", res.get(0));
    Assert.assertEquals("2\t2000\taa", res.get(1));
    // Compact
    CompactionRequest compactionRequest = new CompactionRequest("default", tblName, CompactionType.MAJOR);
    compactionRequest.setPartitionname("part=aa");
    txnHandler.compact(compactionRequest);
    runWorker(hiveConf);
    runCleaner(hiveConf);
    // Verify successful compaction
    List<ShowCompactResponseElement> compacts = txnHandler.showCompact(new ShowCompactRequest()).getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, compacts.get(0).getState());
    // Validate data after compaction
    res = runStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a");
    Assert.assertEquals(2, res.size());
    Assert.assertEquals("1\tNULL\taa", res.get(0));
    Assert.assertEquals("2\t2000\taa", res.get(1));
}
Also used : ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Test(org.junit.Test)

Example 17 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionTxnHandler method testFindNextToCompact2.

@Test
public void testFindNextToCompact2() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=yesterday");
    txnHandler.compact(rqst);
    long now = System.currentTimeMillis();
    boolean expectToday = false;
    CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    if ("ds=today".equals(ci.partName))
        expectToday = false;
    else if ("ds=yesterday".equals(ci.partName))
        expectToday = true;
    else
        fail("partition name should have been today or yesterday but was " + ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);
    ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    if (expectToday)
        assertEquals("ds=today", ci.partName);
    else
        assertEquals("ds=yesterday", ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);
    assertNull(txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)));
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(2, compacts.size());
    for (ShowCompactResponseElement e : compacts) {
        assertEquals("working", e.getState());
        assertTrue(e.getStart() - 5000 < now && e.getStart() + 5000 > now);
        assertEquals("fred", e.getWorkerid());
    }
}
Also used : ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Test(org.junit.Test)

Example 18 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionTxnHandler method testMarkCompacted.

@Test
public void testMarkCompacted() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    txnHandler.markCompacted(ci);
    assertNull(txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)));
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MINOR, c.getType());
    assertEquals("ready for cleaning", c.getState());
    assertNull(c.getWorkerid());
}
Also used : ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Test(org.junit.Test)

Example 19 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestDeltaFilesMetrics method testDeltaFileMetricMultiPartitionedTable.

@Test
public void testDeltaFileMetricMultiPartitionedTable() throws Exception {
    String dbName = "default";
    String tblName = "dp";
    String part1Name = "ds=part1";
    String part2Name = "ds=part2";
    String part3Name = "ds=part3";
    Table t = newTable(dbName, tblName, true);
    List<LockComponent> components = new ArrayList<>();
    Partition p1 = newPartition(t, "part1");
    addDeltaFile(t, p1, 1L, 2L, 2);
    addDeltaFile(t, p1, 3L, 4L, 4);
    Partition p2 = newPartition(t, "part2");
    addBaseFile(t, p2, 5L, 20);
    addDeltaFile(t, p2, 6L, 7L, 2);
    addDeltaFile(t, p2, 8L, 9L, 3);
    addDeltaFile(t, p2, 10L, 11L, 1);
    Partition p3 = newPartition(t, "part3");
    addDeltaFile(t, p3, 12L, 13L, 3);
    addDeltaFile(t, p3, 14L, 15L, 20);
    addDeltaFile(t, p3, 16L, 17L, 50);
    addDeltaFile(t, p3, 18L, 19L, 2);
    components.add(createLockComponent(dbName, tblName, part1Name));
    components.add(createLockComponent(dbName, tblName, part2Name));
    components.add(createLockComponent(dbName, tblName, part3Name));
    burnThroughTransactions(dbName, tblName, 19);
    long txnId = openTxn();
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnId);
    LockResponse res = txnHandler.lock(req);
    Assert.assertEquals(LockState.ACQUIRED, res.getState());
    allocateWriteId(dbName, tblName, txnId);
    txnHandler.commitTxn(new CommitTxnRequest(txnId));
    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 2);
    HiveConf.setFloatVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD, 0.4f);
    startInitiator();
    TimeUnit.SECONDS.sleep(2);
    verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 3, dbName + "." + tblName + Path.SEPARATOR + part3Name, 4), MetricsConstants.COMPACTION_NUM_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part2Name, 2), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
    ShowCompactResponse showCompactResponse = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = showCompactResponse.getCompacts();
    Assert.assertEquals(2, compacts.size());
    // Need to run two worker sessions, to compact all resources in the compaction queue
    startWorker();
    startWorker();
    TimeUnit.SECONDS.sleep(2);
    verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 1), MetricsConstants.COMPACTION_NUM_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part2Name, 3, dbName + "." + tblName + Path.SEPARATOR + part3Name, 4), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
    startCleaner();
    startCleaner();
    TimeUnit.SECONDS.sleep(2);
    verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 1), MetricsConstants.COMPACTION_NUM_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
    verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
}
Also used : CommitTxnRequest(org.apache.hadoop.hive.metastore.api.CommitTxnRequest) Partition(org.apache.hadoop.hive.metastore.api.Partition) Table(org.apache.hadoop.hive.metastore.api.Table) LockComponent(org.apache.hadoop.hive.metastore.api.LockComponent) ArrayList(java.util.ArrayList) LockResponse(org.apache.hadoop.hive.metastore.api.LockResponse) ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) LockRequest(org.apache.hadoop.hive.metastore.api.LockRequest) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Test(org.junit.Test)

Example 20 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestWorker method majorWithAborted.

@Test
public void majorWithAborted() throws Exception {
    LOG.debug("Starting majorWithAborted");
    Table t = newTable("default", "mtwb", false);
    addBaseFile(t, null, 20L, 20);
    addDeltaFile(t, null, 21L, 22L, 2);
    addDeltaFile(t, null, 23L, 25L, 3);
    addLengthFile(t, null, 23L, 25L, 3);
    addDeltaFile(t, null, 26L, 27L, 2);
    burnThroughTransactions("default", "mtwb", 27, null, new HashSet<Long>(Arrays.asList(24L, 25L)));
    CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MAJOR);
    txnHandler.compact(rqst);
    startWorker();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
    // There should still now be 5 directories in the location
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation()));
    Assert.assertEquals(5, stat.length);
    // Find the new delta file and make sure it has the right contents
    Arrays.sort(stat);
    Assert.assertEquals("base_0000027_v0000028", stat[0].getPath().getName());
    Assert.assertEquals("base_20", stat[1].getPath().getName());
    Assert.assertEquals(makeDeltaDirName(21, 22), stat[2].getPath().getName());
    Assert.assertEquals(makeDeltaDirName(23, 25), stat[3].getPath().getName());
    Assert.assertEquals(makeDeltaDirName(26, 27), stat[4].getPath().getName());
}
Also used : Path(org.apache.hadoop.fs.Path) Table(org.apache.hadoop.hive.metastore.api.Table) FileStatus(org.apache.hadoop.fs.FileStatus) ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) FileSystem(org.apache.hadoop.fs.FileSystem) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Test(org.junit.Test)

Aggregations

ShowCompactResponseElement (org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement)100 ShowCompactResponse (org.apache.hadoop.hive.metastore.api.ShowCompactResponse)87 Test (org.junit.Test)81 ShowCompactRequest (org.apache.hadoop.hive.metastore.api.ShowCompactRequest)79 Table (org.apache.hadoop.hive.metastore.api.Table)58 CompactionRequest (org.apache.hadoop.hive.metastore.api.CompactionRequest)46 ArrayList (java.util.ArrayList)42 Partition (org.apache.hadoop.hive.metastore.api.Partition)27 LockComponent (org.apache.hadoop.hive.metastore.api.LockComponent)26 LockRequest (org.apache.hadoop.hive.metastore.api.LockRequest)26 FileSystem (org.apache.hadoop.fs.FileSystem)25 LockResponse (org.apache.hadoop.hive.metastore.api.LockResponse)25 Path (org.apache.hadoop.fs.Path)24 FileStatus (org.apache.hadoop.fs.FileStatus)20 CommitTxnRequest (org.apache.hadoop.hive.metastore.api.CommitTxnRequest)17 TxnStore (org.apache.hadoop.hive.metastore.txn.TxnStore)16 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)11 CompactionInfo (org.apache.hadoop.hive.metastore.txn.CompactionInfo)11 IOException (java.io.IOException)9 List (java.util.List)9