use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.
the class TestCleaner method notBlockedBySubsequentLock.
@Test
public void notBlockedBySubsequentLock() throws Exception {
Table t = newTable("default", "bblt", false);
// Set the run frequency low on this test so it doesn't take long
conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100, TimeUnit.MILLISECONDS);
addBaseFile(t, null, 20L, 20);
addDeltaFile(t, null, 21L, 22L, 2);
addDeltaFile(t, null, 23L, 24L, 2);
addDeltaFile(t, null, 21L, 24L, 4);
burnThroughTransactions("default", "bblt", 25);
CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR);
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact("fred");
txnHandler.markCompacted(ci);
txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
comp.setTablename("bblt");
comp.setOperationType(DataOperationType.INSERT);
List<LockComponent> components = new ArrayList<LockComponent>(1);
components.add(comp);
LockRequest req = new LockRequest(components, "me", "localhost");
LockResponse res = txnHandler.lock(req);
AtomicBoolean looped = new AtomicBoolean();
looped.set(false);
startCleaner(looped);
// Make sure the compactor has a chance to run once
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
// There should still be one request, as the locks still held.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
Assert.assertEquals(1, compacts.size());
// obtain a second lock. This shouldn't block cleaner as it was acquired after the initial
// clean request
LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
comp2.setTablename("bblt");
comp.setOperationType(DataOperationType.SELECT);
List<LockComponent> components2 = new ArrayList<LockComponent>(1);
components2.add(comp2);
LockRequest req2 = new LockRequest(components, "me", "localhost");
LockResponse res2 = txnHandler.lock(req2);
// Unlock the previous lock
txnHandler.unlock(new UnlockRequest(res.getLockid()));
looped.set(false);
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
stopThread();
Thread.currentThread().sleep(200);
// Check there are no compactions requests left.
rsp = txnHandler.showCompact(new ShowCompactRequest());
compacts = rsp.getCompacts();
Assert.assertEquals(1, compacts.size());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.
the class TestCleaner method cleanupAfterMajorPartitionCompactionNoBase.
@Test
public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception {
Table t = newTable("default", "campcnb", true);
Partition p = newPartition(t, "today");
addDeltaFile(t, p, 1L, 22L, 22);
addDeltaFile(t, p, 23L, 24L, 2);
addBaseFile(t, p, 25L, 25);
burnThroughTransactions("default", "campcnb", 25);
CompactionRequest rqst = new CompactionRequest("default", "campcnb", CompactionType.MAJOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact("fred");
txnHandler.markCompacted(ci);
txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
startCleaner();
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(1, rsp.getCompactsSize());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, p);
Assert.assertEquals(1, paths.size());
Assert.assertEquals("base_25", paths.get(0).getName());
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.
the class TestCleaner method cleanupAfterMajorTableCompaction.
@Test
public void cleanupAfterMajorTableCompaction() throws Exception {
Table t = newTable("default", "camtc", false);
addBaseFile(t, null, 20L, 20);
addDeltaFile(t, null, 21L, 22L, 2);
addDeltaFile(t, null, 23L, 24L, 2);
addBaseFile(t, null, 25L, 25);
burnThroughTransactions("default", "camtc", 25);
CompactionRequest rqst = new CompactionRequest("default", "camtc", CompactionType.MAJOR);
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact("fred");
txnHandler.markCompacted(ci);
txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
startCleaner();
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(1, rsp.getCompactsSize());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, null);
Assert.assertEquals(1, paths.size());
Assert.assertEquals("base_25", paths.get(0).getName());
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.
the class TestCleaner method blockedByLockTable.
@Test
public void blockedByLockTable() throws Exception {
Table t = newTable("default", "bblt", false);
addBaseFile(t, null, 20L, 20);
addDeltaFile(t, null, 21L, 22L, 2);
addDeltaFile(t, null, 23L, 24L, 2);
addDeltaFile(t, null, 21L, 24L, 4);
burnThroughTransactions("default", "bblt", 25);
CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR);
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact("fred");
txnHandler.markCompacted(ci);
txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
comp.setTablename("bblt");
comp.setOperationType(DataOperationType.SELECT);
List<LockComponent> components = new ArrayList<LockComponent>(1);
components.add(comp);
LockRequest req = new LockRequest(components, "me", "localhost");
LockResponse res = txnHandler.lock(req);
startCleaner();
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
Assert.assertEquals(1, compacts.size());
Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
Assert.assertEquals("bblt", compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MINOR, compacts.get(0).getType());
}
use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.
the class TestCleaner method droppedTable.
@Test
public void droppedTable() throws Exception {
Table t = newTable("default", "dt", false);
addDeltaFile(t, null, 1L, 22L, 22);
addDeltaFile(t, null, 23L, 24L, 2);
addBaseFile(t, null, 25L, 25);
burnThroughTransactions("default", "dt", 25);
CompactionRequest rqst = new CompactionRequest("default", "dt", CompactionType.MINOR);
txnHandler.compact(rqst);
CompactionInfo ci = txnHandler.findNextToCompact("fred");
txnHandler.markCompacted(ci);
txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
// Drop table will clean the table entry from the compaction queue and hence cleaner have no effect
ms.dropTable("default", "dt");
startCleaner();
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0, rsp.getCompactsSize());
}
Aggregations