Examples with ShowCompactResponseElement org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement used on opensource projects

Example 41 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestInitiator method recoverFailedRemoteWorkers.

@Test
public void recoverFailedRemoteWorkers() throws Exception {
    Table t = newTable("default", "rfrw1", false);
    CompactionRequest rqst = new CompactionRequest("default", "rfrw1", CompactionType.MINOR);
    txnHandler.compact(rqst);
    txnHandler.findNextToCompact(aFindNextCompactRequest("nosuchhost-193892", "4.0.0"));
    conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS);
    startInitiator();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("initiated", compacts.get(0).getState());
}

Example 42 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestInitiator method testInitiatorFailure.

@Test
public void testInitiatorFailure() throws Exception {
    String tableName = "my_table";
    Table t = newTable("default", tableName, false);
    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 1);
    // 2 aborts
    for (int i = 0; i < 2; i++) {
        long txnid = openTxn();
        LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
        comp.setTablename(tableName);
        comp.setOperationType(DataOperationType.UPDATE);
        List<LockComponent> components = new ArrayList<LockComponent>(1);
        components.add(comp);
        LockRequest req = new LockRequest(components, "me", "localhost");
        req.setTxnid(txnid);
        LockResponse res = txnHandler.lock(req);
        txnHandler.abortTxn(new AbortTxnRequest(txnid));
    }
    // run and fail initiator
    Initiator initiator = Mockito.spy(new Initiator());
    initiator.setThreadId((int) t.getId());
    initiator.setConf(conf);
    initiator.init(new AtomicBoolean(true));
    doThrow(new RuntimeException("This was thrown on purpose by testInitiatorFailure")).when(initiator).resolveTable(any());
    initiator.run();
    // verify status of table compaction
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("did not initiate", compacts.get(0).getState());
    Assert.assertEquals(tableName, compacts.get(0).getTablename());
}

Example 43 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestInitiator method testInitiatorHostAndVersion.

@Test
public void testInitiatorHostAndVersion() throws Exception {
    String tableName = "my_table";
    Table t = newTable("default", tableName, false);
    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 1);
    // 2 aborts
    for (int i = 0; i < 2; i++) {
        long txnid = openTxn();
        LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
        comp.setTablename(tableName);
        comp.setOperationType(DataOperationType.UPDATE);
        List<LockComponent> components = new ArrayList<>(1);
        components.add(comp);
        LockRequest req = new LockRequest(components, "me", "localhost");
        req.setTxnid(txnid);
        txnHandler.lock(req);
        txnHandler.abortTxn(new AbortTxnRequest(txnid));
    }
    // need to mock the runtime version, because the manifest file won't be there in the mvn test setup
    Initiator initiator = Mockito.spy(new Initiator());
    initiator.setThreadId((int) t.getId());
    initiator.setConf(conf);
    String runtimeVersion = "4.0.0-SNAPSHOT";
    doReturn(runtimeVersion).when(initiator).getRuntimeVersion();
    initiator.init(new AtomicBoolean(true));
    initiator.run();
    // verify status of table compaction
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("initiated", compacts.get(0).getState());
    Assert.assertEquals(tableName, compacts.get(0).getTablename());
    Assert.assertEquals(runtimeVersion, compacts.get(0).getInitiatorVersion());
    // split the threadid
    String[] parts = compacts.get(0).getInitiatorId().split("-");
    Assert.assertTrue(parts.length > 1);
    Assert.assertEquals(ServerUtils.hostname(), String.join("-", Arrays.copyOfRange(parts, 0, parts.length - 1)));
}

Example 44 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCleaner method blockedByLockPartition.

@Test
public void blockedByLockPartition() throws Exception {
    Table t = newTable("default", "bblp", true);
    Partition p = newPartition(t, "today");
    addBaseFile(t, p, 20L, 20);
    addDeltaFile(t, p, 21L, 22L, 2);
    addDeltaFile(t, p, 23L, 24L, 2);
    addDeltaFile(t, p, 21L, 24L, 4);
    burnThroughTransactions("default", "bblp", 25);
    CompactionRequest rqst = new CompactionRequest("default", "bblp", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    txnHandler.markCompacted(ci);
    txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default");
    comp.setTablename("bblp");
    comp.setPartitionname("ds=today");
    comp.setOperationType(DataOperationType.DELETE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    OpenTxnsResponse resp = txnHandler.openTxns(new OpenTxnRequest(1, "Dracula", "Transylvania"));
    req.setTxnid(resp.getTxn_ids().get(0));
    LockResponse res = txnHandler.lock(req);
    startCleaner();
    // Check there are no compactions requests left.
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
    Assert.assertEquals("bblp", compacts.get(0).getTablename());
    Assert.assertEquals("ds=today", compacts.get(0).getPartitionname());
    Assert.assertEquals(CompactionType.MINOR, compacts.get(0).getType());
}

Example 45 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCleaner method notBlockedBySubsequentLock.

@Test
public void notBlockedBySubsequentLock() throws Exception {
    Table t = newTable("default", "bblt", false);
    // Set the run frequency low on this test so it doesn't take long
    conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100, TimeUnit.MILLISECONDS);
    addBaseFile(t, null, 20L, 20);
    addDeltaFile(t, null, 21L, 22L, 2);
    addDeltaFile(t, null, 23L, 24L, 2);
    addDeltaFile(t, null, 21L, 24L, 4);
    burnThroughTransactions("default", "bblt", 25);
    CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR);
    txnHandler.compact(rqst);
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    txnHandler.markCompacted(ci);
    txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
    comp.setTablename("bblt");
    comp.setOperationType(DataOperationType.INSERT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    AtomicBoolean looped = new AtomicBoolean();
    looped.set(false);
    startCleaner(looped);
    // Make sure the compactor has a chance to run once
    while (!looped.get()) {
        Thread.currentThread().sleep(100);
    }
    // There should still be one request, as the locks still held.
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    // obtain a second lock.  This shouldn't block cleaner as it was acquired after the initial
    // clean request
    LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
    comp2.setTablename("bblt");
    comp.setOperationType(DataOperationType.SELECT);
    List<LockComponent> components2 = new ArrayList<LockComponent>(1);
    components2.add(comp2);
    LockRequest req2 = new LockRequest(components, "me", "localhost");
    LockResponse res2 = txnHandler.lock(req2);
    // Unlock the previous lock
    txnHandler.unlock(new UnlockRequest(res.getLockid()));
    looped.set(false);
    while (!looped.get()) {
        Thread.currentThread().sleep(100);
    }
    stopThread();
    Thread.currentThread().sleep(200);
    // Check there are no compactions requests left.
    rsp = txnHandler.showCompact(new ShowCompactRequest());
    compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}