Examples with ShowCompactResponse org.apache.hadoop.hive.metastore.api.ShowCompactResponse used on opensource projects

Example 16 with ShowCompactResponse

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.

the class TestCleaner method droppedPartition.

@Test
public void droppedPartition() throws Exception {
    Table t = newTable("default", "dp", true);
    Partition p = newPartition(t, "today");
    addDeltaFile(t, p, 1L, 22L, 22);
    addDeltaFile(t, p, 23L, 24L, 2);
    addBaseFile(t, p, 25L, 25);
    burnThroughTransactions(25);
    CompactionRequest rqst = new CompactionRequest("default", "dp", CompactionType.MAJOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    txnHandler.markCompacted(ci);
    txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
    ms.dropPartition("default", "dp", Collections.singletonList("today"), true);
    startCleaner();
    // Check there are no compactions requests left.
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals(1, rsp.getCompactsSize());
    Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}

Example 17 with ShowCompactResponse

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.

the class TestInitiator method compactPartitionHighDeltaPct.

@Test
public void compactPartitionHighDeltaPct() throws Exception {
    Table t = newTable("default", "cphdp", true);
    Partition p = newPartition(t, "today");
    addBaseFile(t, p, 20L, 20);
    addDeltaFile(t, p, 21L, 22L, 2);
    addDeltaFile(t, p, 23L, 24L, 2);
    burnThroughTransactions(23);
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default");
    comp.setTablename("cphdp");
    comp.setPartitionname("ds=today");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    txnHandler.commitTxn(new CommitTxnRequest(txnid));
    startInitiator();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("initiated", compacts.get(0).getState());
    Assert.assertEquals("cphdp", compacts.get(0).getTablename());
    Assert.assertEquals("ds=today", compacts.get(0).getPartitionname());
    Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType());
}

Example 18 with ShowCompactResponse

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.

the class TestInitiator method noCompactWhenNoCompactSet.

@Test
public void noCompactWhenNoCompactSet() throws Exception {
    Map<String, String> parameters = new HashMap<String, String>(1);
    parameters.put("NO_AUTO_COMPACTION", "true");
    Table t = newTable("default", "ncwncs", false, parameters);
    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
    for (int i = 0; i < 11; i++) {
        long txnid = openTxn();
        LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
        comp.setTablename("ncwncs");
        comp.setOperationType(DataOperationType.UPDATE);
        List<LockComponent> components = new ArrayList<LockComponent>(1);
        components.add(comp);
        LockRequest req = new LockRequest(components, "me", "localhost");
        req.setTxnid(txnid);
        LockResponse res = txnHandler.lock(req);
        txnHandler.abortTxn(new AbortTxnRequest(txnid));
    }
    startInitiator();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals(0, rsp.getCompactsSize());
}

Example 19 with ShowCompactResponse

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.

the class TestTxnCommands2 method testOriginalFileReaderWhenNonAcidConvertedToAcid.

@Test
public void testOriginalFileReaderWhenNonAcidConvertedToAcid() throws Exception {
    // 1. Insert five rows to Non-ACID table.
    runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,2),(3,4),(5,6),(7,8),(9,10)");
    // 2. Convert NONACIDORCTBL to ACID table.
    runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true')");
    runStatementOnDriver("update " + Table.NONACIDORCTBL + " set b = b*2 where b in (4,10)");
    runStatementOnDriver("delete from " + Table.NONACIDORCTBL + " where a = 7");
    List<String> rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL + " order by a,b");
    int[][] resultData = new int[][] { { 1, 2 }, { 3, 8 }, { 5, 6 }, { 9, 20 } };
    Assert.assertEquals(stringifyValues(resultData), rs);
    // 3. Perform a major compaction.
    runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " compact 'MAJOR'");
    runWorker(hiveConf);
    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
    Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
    Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
    Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local"));
    // 3. Perform a delete.
    runStatementOnDriver("delete from " + Table.NONACIDORCTBL + " where a = 1");
    rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL + " order by a,b");
    resultData = new int[][] { { 3, 8 }, { 5, 6 }, { 9, 20 } };
    Assert.assertEquals(stringifyValues(resultData), rs);
}

Example 20 with ShowCompactResponse

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponse in project hive by apache.

the class DDLTask method showCompactions.

private int showCompactions(Hive db, ShowCompactionsDesc desc) throws HiveException {
    // Call the metastore to get the status of all known compactions (completed get purged eventually)
    ShowCompactResponse rsp = db.showCompactions();
    // Write the results into the file
    final String noVal = " --- ";
    DataOutputStream os = getOutputStream(desc.getResFile());
    try {
        // Write a header
        os.writeBytes("Database");
        os.write(separator);
        os.writeBytes("Table");
        os.write(separator);
        os.writeBytes("Partition");
        os.write(separator);
        os.writeBytes("Type");
        os.write(separator);
        os.writeBytes("State");
        os.write(separator);
        os.writeBytes("Worker");
        os.write(separator);
        os.writeBytes("Start Time");
        os.write(separator);
        os.writeBytes("Duration(ms)");
        os.write(separator);
        os.writeBytes("HadoopJobId");
        os.write(terminator);
        if (rsp.getCompacts() != null) {
            for (ShowCompactResponseElement e : rsp.getCompacts()) {
                os.writeBytes(e.getDbname());
                os.write(separator);
                os.writeBytes(e.getTablename());
                os.write(separator);
                String part = e.getPartitionname();
                os.writeBytes(part == null ? noVal : part);
                os.write(separator);
                os.writeBytes(e.getType().toString());
                os.write(separator);
                os.writeBytes(e.getState());
                os.write(separator);
                String wid = e.getWorkerid();
                os.writeBytes(wid == null ? noVal : wid);
                os.write(separator);
                os.writeBytes(e.isSetStart() ? Long.toString(e.getStart()) : noVal);
                os.write(separator);
                os.writeBytes(e.isSetEndTime() ? Long.toString(e.getEndTime() - e.getStart()) : noVal);
                os.write(separator);
                os.writeBytes(e.isSetHadoopJobId() ? e.getHadoopJobId() : noVal);
                os.write(terminator);
            }
        }
    } catch (IOException e) {
        LOG.warn("show compactions: " + stringifyException(e));
        return 1;
    } finally {
        IOUtils.closeStream(os);
    }
    return 0;
}