use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.
the class TestTxnCommands2 method testSchemaEvolutionCompaction.
/**
* Create a table with schema evolution, and verify that no data is lost during (MR major)
* compaction.
*
* @throws Exception if a query fails
*/
@Test
public void testSchemaEvolutionCompaction() throws Exception {
String tblName = "schemaevolutioncompaction";
runStatementOnDriver("drop table if exists " + tblName);
runStatementOnDriver("CREATE TABLE " + tblName + "(a INT) " + " PARTITIONED BY(part string)" + " STORED AS ORC TBLPROPERTIES ('transactional'='true')");
// First INSERT round.
runStatementOnDriver("insert into " + tblName + " partition (part='aa') values (1)");
// ALTER TABLE ... ADD COLUMNS
runStatementOnDriver("ALTER TABLE " + tblName + " ADD COLUMNS(b int)");
// Second INSERT round.
runStatementOnDriver("insert into " + tblName + " partition (part='aa') values (2, 2000)");
// Validate data
List<String> res = runStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a");
Assert.assertEquals(2, res.size());
Assert.assertEquals("1\tNULL\taa", res.get(0));
Assert.assertEquals("2\t2000\taa", res.get(1));
// Compact
CompactionRequest compactionRequest = new CompactionRequest("default", tblName, CompactionType.MAJOR);
compactionRequest.setPartitionname("part=aa");
txnHandler.compact(compactionRequest);
runWorker(hiveConf);
runCleaner(hiveConf);
// Verify successful compaction
List<ShowCompactResponseElement> compacts = txnHandler.showCompact(new ShowCompactRequest()).getCompacts();
Assert.assertEquals(1, compacts.size());
Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, compacts.get(0).getState());
// Validate data after compaction
res = runStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a");
Assert.assertEquals(2, res.size());
Assert.assertEquals("1\tNULL\taa", res.get(0));
Assert.assertEquals("2\t2000\taa", res.get(1));
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.
the class TestCompactionTxnHandler method testFindNextToCompact2.
@Test
public void testFindNextToCompact2() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=yesterday");
txnHandler.compact(rqst);
long now = System.currentTimeMillis();
boolean expectToday = false;
CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
assertNotNull(ci);
assertEquals("foo", ci.dbname);
assertEquals("bar", ci.tableName);
if ("ds=today".equals(ci.partName))
expectToday = false;
else if ("ds=yesterday".equals(ci.partName))
expectToday = true;
else
fail("partition name should have been today or yesterday but was " + ci.partName);
assertEquals(CompactionType.MINOR, ci.type);
ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
assertNotNull(ci);
assertEquals("foo", ci.dbname);
assertEquals("bar", ci.tableName);
if (expectToday)
assertEquals("ds=today", ci.partName);
else
assertEquals("ds=yesterday", ci.partName);
assertEquals(CompactionType.MINOR, ci.type);
assertNull(txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)));
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
assertEquals(2, compacts.size());
for (ShowCompactResponseElement e : compacts) {
assertEquals("working", e.getState());
assertTrue(e.getStart() - 5000 < now && e.getStart() + 5000 > now);
assertEquals("fred", e.getWorkerid());
}
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.
the class TestCompactionTxnHandler method testMarkCompacted.
@Test
public void testMarkCompacted() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
assertNotNull(ci);
txnHandler.markCompacted(ci);
assertNull(txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)));
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
assertEquals(1, compacts.size());
ShowCompactResponseElement c = compacts.get(0);
assertEquals("foo", c.getDbname());
assertEquals("bar", c.getTablename());
assertEquals("ds=today", c.getPartitionname());
assertEquals(CompactionType.MINOR, c.getType());
assertEquals("ready for cleaning", c.getState());
assertNull(c.getWorkerid());
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.
the class TestDeltaFilesMetrics method testDeltaFileMetricMultiPartitionedTable.
@Test
public void testDeltaFileMetricMultiPartitionedTable() throws Exception {
String dbName = "default";
String tblName = "dp";
String part1Name = "ds=part1";
String part2Name = "ds=part2";
String part3Name = "ds=part3";
Table t = newTable(dbName, tblName, true);
List<LockComponent> components = new ArrayList<>();
Partition p1 = newPartition(t, "part1");
addDeltaFile(t, p1, 1L, 2L, 2);
addDeltaFile(t, p1, 3L, 4L, 4);
Partition p2 = newPartition(t, "part2");
addBaseFile(t, p2, 5L, 20);
addDeltaFile(t, p2, 6L, 7L, 2);
addDeltaFile(t, p2, 8L, 9L, 3);
addDeltaFile(t, p2, 10L, 11L, 1);
Partition p3 = newPartition(t, "part3");
addDeltaFile(t, p3, 12L, 13L, 3);
addDeltaFile(t, p3, 14L, 15L, 20);
addDeltaFile(t, p3, 16L, 17L, 50);
addDeltaFile(t, p3, 18L, 19L, 2);
components.add(createLockComponent(dbName, tblName, part1Name));
components.add(createLockComponent(dbName, tblName, part2Name));
components.add(createLockComponent(dbName, tblName, part3Name));
burnThroughTransactions(dbName, tblName, 19);
long txnId = openTxn();
LockRequest req = new LockRequest(components, "me", "localhost");
req.setTxnid(txnId);
LockResponse res = txnHandler.lock(req);
Assert.assertEquals(LockState.ACQUIRED, res.getState());
allocateWriteId(dbName, tblName, txnId);
txnHandler.commitTxn(new CommitTxnRequest(txnId));
HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 2);
HiveConf.setFloatVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD, 0.4f);
startInitiator();
TimeUnit.SECONDS.sleep(2);
verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 3, dbName + "." + tblName + Path.SEPARATOR + part3Name, 4), MetricsConstants.COMPACTION_NUM_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part2Name, 2), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
ShowCompactResponse showCompactResponse = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = showCompactResponse.getCompacts();
Assert.assertEquals(2, compacts.size());
// Need to run two worker sessions, to compact all resources in the compaction queue
startWorker();
startWorker();
TimeUnit.SECONDS.sleep(2);
verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 1), MetricsConstants.COMPACTION_NUM_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part2Name, 3, dbName + "." + tblName + Path.SEPARATOR + part3Name, 4), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
startCleaner();
startCleaner();
TimeUnit.SECONDS.sleep(2);
verifyDeltaMetricsMatch(ImmutableMap.of(dbName + "." + tblName + Path.SEPARATOR + part1Name, 2, dbName + "." + tblName + Path.SEPARATOR + part2Name, 1), MetricsConstants.COMPACTION_NUM_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_SMALL_DELTAS);
verifyDeltaMetricsMatch(ImmutableMap.of(), MetricsConstants.COMPACTION_NUM_OBSOLETE_DELTAS);
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.
the class TestWorker method majorWithAborted.
@Test
public void majorWithAborted() throws Exception {
LOG.debug("Starting majorWithAborted");
Table t = newTable("default", "mtwb", false);
addBaseFile(t, null, 20L, 20);
addDeltaFile(t, null, 21L, 22L, 2);
addDeltaFile(t, null, 23L, 25L, 3);
addLengthFile(t, null, 23L, 25L, 3);
addDeltaFile(t, null, 26L, 27L, 2);
burnThroughTransactions("default", "mtwb", 27, null, new HashSet<Long>(Arrays.asList(24L, 25L)));
CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MAJOR);
txnHandler.compact(rqst);
startWorker();
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
Assert.assertEquals(1, compacts.size());
Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
// There should still now be 5 directories in the location
FileSystem fs = FileSystem.get(conf);
FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(5, stat.length);
// Find the new delta file and make sure it has the right contents
Arrays.sort(stat);
Assert.assertEquals("base_0000027_v0000028", stat[0].getPath().getName());
Assert.assertEquals("base_20", stat[1].getPath().getName());
Assert.assertEquals(makeDeltaDirName(21, 22), stat[2].getPath().getName());
Assert.assertEquals(makeDeltaDirName(23, 25), stat[3].getPath().getName());
Assert.assertEquals(makeDeltaDirName(26, 27), stat[4].getPath().getName());
}
Aggregations