Examples with ShowCompactResponseElement org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement used on opensource projects

Example 81 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestInitiator method chooseMajorOverMinorWhenBothValid.

@Test
public void chooseMajorOverMinorWhenBothValid() throws Exception {
    Table t = newTable("default", "cmomwbv", false);
    addBaseFile(t, null, 200L, 200);
    addDeltaFile(t, null, 201L, 211L, 11);
    addDeltaFile(t, null, 212L, 222L, 11);
    addDeltaFile(t, null, 223L, 233L, 11);
    addDeltaFile(t, null, 234L, 244L, 11);
    addDeltaFile(t, null, 245L, 255L, 11);
    addDeltaFile(t, null, 256L, 266L, 11);
    addDeltaFile(t, null, 267L, 277L, 11);
    addDeltaFile(t, null, 278L, 288L, 11);
    addDeltaFile(t, null, 289L, 299L, 11);
    addDeltaFile(t, null, 300L, 310L, 11);
    addDeltaFile(t, null, 311L, 321L, 11);
    burnThroughTransactions("default", "cmomwbv", 320);
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
    comp.setTablename("cmomwbv");
    comp.setOperationType(DataOperationType.UPDATE);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    long writeid = allocateWriteId("default", "cmomwbv", txnid);
    Assert.assertEquals(321, writeid);
    txnHandler.commitTxn(new CommitTxnRequest(txnid));
    startInitiator();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(1, compacts.size());
    Assert.assertEquals("initiated", compacts.get(0).getState());
    Assert.assertEquals("cmomwbv", compacts.get(0).getTablename());
    Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType());
}

Example 82 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetrics method testCleaningAgeMetricsOrder.

@Test
public void testCleaningAgeMetricsOrder() {
    ShowCompactResponse scr = new ShowCompactResponse();
    long start = System.currentTimeMillis();
    List<ShowCompactResponseElement> elements = ImmutableList.of(generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, start, false, "4.0.0", "4.0.0", -1L, start - 1_000L), generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, start, false, "4.0.0", "4.0.0", -1L, start - 15_000L));
    scr.setCompacts(elements);
    AcidMetricService.updateMetricsFromShowCompact(scr);
    // Check that the age is older than 10s
    Assert.assertTrue(Metrics.getOrCreateGauge(MetricsConstants.COMPACTION_OLDEST_CLEANING_AGE).intValue() > 10);
    // Check the reverse order
    elements = ImmutableList.of(generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, start, false, "4.0.0", "4.0.0", -1L, start - 25_000L), generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, start, false, "4.0.0", "4.0.0", -1L, start - 1_000L));
    scr.setCompacts(elements);
    AcidMetricService.updateMetricsFromShowCompact(scr);
    // Check that the age is older than 20s
    Assert.assertTrue(Metrics.getOrCreateGauge(MetricsConstants.COMPACTION_OLDEST_CLEANING_AGE).intValue() > 20);
}

Example 83 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetrics method testCleaningAgeMetrics.

@Test
public void testCleaningAgeMetrics() {
    ShowCompactResponse scr = new ShowCompactResponse();
    long start = System.currentTimeMillis() - 1000L;
    List<ShowCompactResponseElement> elements = ImmutableList.of(generateElement(19, "db3", "tb7", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, System.currentTimeMillis(), true, "4.0.0", "4.0.0", -1L, start));
    scr.setCompacts(elements);
    AcidMetricService.updateMetricsFromShowCompact(scr);
    long diff = (System.currentTimeMillis() - start) / 1000;
    // Check that we have at least 1s old compaction age, but not more than expected
    int age = Metrics.getOrCreateGauge(MetricsConstants.COMPACTION_OLDEST_CLEANING_AGE).intValue();
    Assert.assertTrue(age <= diff);
    Assert.assertTrue(age >= 1);
}

Example 84 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetrics method testInitiatedAgeMetricsOrder.

@Test
public void testInitiatedAgeMetricsOrder() {
    ShowCompactResponse scr = new ShowCompactResponse();
    long start = System.currentTimeMillis();
    List<ShowCompactResponseElement> elements = ImmutableList.of(generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.INITIATED_RESPONSE, start - 1_000L), generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.INITIATED_RESPONSE, start - 15_000L));
    scr.setCompacts(elements);
    AcidMetricService.updateMetricsFromShowCompact(scr);
    // Check that the age is older than 10s
    Assert.assertTrue(Metrics.getOrCreateGauge(MetricsConstants.COMPACTION_OLDEST_ENQUEUE_AGE).intValue() > 10);
    // Check the reverse order
    elements = ImmutableList.of(generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.INITIATED_RESPONSE, start - 25_000L), generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.INITIATED_RESPONSE, start - 1_000L));
    scr.setCompacts(elements);
    AcidMetricService.updateMetricsFromShowCompact(scr);
    // Check that the age is older than 20s
    Assert.assertTrue(Metrics.getOrCreateGauge(MetricsConstants.COMPACTION_OLDEST_ENQUEUE_AGE).intValue() > 20);
}

Example 85 with ShowCompactResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement in project hive by apache.

the class TestCompactionMetrics method testInitiatorPerfMetricsEnabled.

@Test
public void testInitiatorPerfMetricsEnabled() throws Exception {
    Metrics.getOrCreateGauge(INITIATED_METRICS_KEY).set(0);
    long initiatorCycles = Objects.requireNonNull(Metrics.getOrCreateTimer(INITIATOR_CYCLE_KEY)).getCount();
    Table t = newTable("default", "ime", true);
    List<LockComponent> components = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        Partition p = newPartition(t, "part" + (i + 1));
        addBaseFile(t, p, 20L, 20);
        addDeltaFile(t, p, 21L, 22L, 2);
        addDeltaFile(t, p, 23L, 24L, 2);
        LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default");
        comp.setTablename("ime");
        comp.setPartitionname("ds=part" + (i + 1));
        comp.setOperationType(DataOperationType.UPDATE);
        components.add(comp);
    }
    burnThroughTransactions("default", "ime", 23);
    long txnid = openTxn();
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    Assert.assertEquals(LockState.ACQUIRED, res.getState());
    long writeid = allocateWriteId("default", "ime", txnid);
    Assert.assertEquals(24, writeid);
    txnHandler.commitTxn(new CommitTxnRequest(txnid));
    startInitiator();
    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    Assert.assertEquals(10, compacts.size());
    Assert.assertEquals(initiatorCycles + 1, Objects.requireNonNull(Metrics.getOrCreateTimer(INITIATOR_CYCLE_KEY)).getCount());
    runAcidMetricService();
    Assert.assertEquals(10, Metrics.getOrCreateGauge(INITIATED_METRICS_KEY).intValue());
}