Examples with ShowCompactResponseElement org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement used on opensource projects

Example 61 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestWorker method minorTableNoBase.

@Test
public void minorTableNoBase() throws Exception {
    LOG.debug("Starting minorTableWithBase");
    Table t = newTable("default", "mtnb", false);
    addDeltaFile(t, null, 1L, 2L, 2);
    addDeltaFile(t, null, 3L, 4L, 2);
    burnThroughTransactions("default", "mtnb", 5);
    CompactionRequest rqst = new CompactionRequest("default", "mtnb", CompactionType.MINOR);
    txnHandler.compact(rqst);
    startWorker();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("ready for cleaning", compacts.get(0).getState());
    // There should still now be 5 directories in the location
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation()));
    Assert.assertEquals(4, stat.length);
    // Find the new delta file and make sure it has the right contents
    boolean sawNewDelta = false;
    for (int i = 0; i < stat.length; i++) {
        if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(1, 4) + "_v0000006")) {
            sawNewDelta = true;
            FileStatus[] buckets = fs.listStatus(stat[i].getPath(), AcidUtils.hiddenFileFilter);
            Assert.assertEquals(2, buckets.length);
            Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
            Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
            Assert.assertEquals(104L, buckets[0].getLen());
            Assert.assertEquals(104L, buckets[1].getLen());
        }
        if (stat[i].getPath().getName().equals(makeDeleteDeltaDirNameCompacted(1, 4) + "_v0000006")) {
            sawNewDelta = true;
            FileStatus[] buckets = fs.listStatus(stat[i].getPath(), AcidUtils.hiddenFileFilter);
            Assert.assertEquals(2, buckets.length);
            Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
            Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
            Assert.assertEquals(104L, buckets[0].getLen());
            Assert.assertEquals(104L, buckets[1].getLen());
        } else {
            LOG.debug("This is not the delta file you are looking for " + stat[i].getPath().getName());
        }
    }
    Assert.assertTrue(toString(stat), sawNewDelta);
}

Example 62 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestWorker method insertOnlyDisabled.

@Test
public void insertOnlyDisabled() throws Exception {
    Map<String, String> parameters = new HashMap<>();
    parameters.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY);
    Table t = newTable("default", "iod", false, parameters);
    addDeltaFile(t, null, 1L, 2L, 2);
    addDeltaFile(t, null, 3L, 4L, 2);
    burnThroughTransactions("default", "iod", 5);
    conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_COMPACT_MM, false);
    CompactionRequest rqst = new CompactionRequest("default", "iod", CompactionType.MINOR);
    txnHandler.compact(rqst);
    startWorker();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("failed", compacts.get(0).getState());
}

Example 63 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionTxnHandler method testFindNextToCompact.

@Test
public void testFindNextToCompact() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    long now = System.currentTimeMillis();
    CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    assertEquals("ds=today", ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);
    assertNull(ci.runAs);
    assertNull(txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)));
    ci.runAs = "bob";
    txnHandler.updateCompactorState(ci, openTxn());
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MINOR, c.getType());
    assertEquals("working", c.getState());
    assertTrue(c.getStart() - 5000 < now && c.getStart() + 5000 > now);
    assertEquals("fred", c.getWorkerid());
    assertEquals("bob", c.getRunAs());
}

Example 64 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionTxnHandler method checkEnqueueTime.

private void checkEnqueueTime(long enqueueTime) throws MetaException {
    ShowCompactResponse showCompactResponse = txnHandler.showCompact(new ShowCompactRequest());
    ShowCompactResponseElement element = showCompactResponse.getCompacts().get(0);
    assertTrue(element.isSetEnqueueTime());
    assertEquals(enqueueTime, element.getEnqueueTime());
}

Example 65 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionTxnHandler method testEnqueueTimeThroughLifeCycle.

@Test
public void testEnqueueTimeThroughLifeCycle() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    CompactionRequest rqst = new CompactionRequest(dbName, tableName, CompactionType.MINOR);
    rqst.setPartitionname(partitionName);
    long before = System.currentTimeMillis();
    txnHandler.compact(rqst);
    long after = System.currentTimeMillis();
    ShowCompactResponse showCompactResponse = txnHandler.showCompact(new ShowCompactRequest());
    ShowCompactResponseElement element = showCompactResponse.getCompacts().get(0);
    assertTrue(element.isSetEnqueueTime());
    long enqueueTime = element.getEnqueueTime();
    assertTrue(enqueueTime <= after);
    assertTrue(enqueueTime >= before);
    CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION));
    ci.runAs = "bob";
    txnHandler.updateCompactorState(ci, openTxn());
    checkEnqueueTime(enqueueTime);
    txnHandler.markCompacted(ci);
    checkEnqueueTime(enqueueTime);
    txnHandler.markCleaned(ci);
    checkEnqueueTime(enqueueTime);
}