Examples with ValidCompactorTxnList org.apache.hadoop.hive.common.ValidCompactorTxnList used on opensource projects

Example 1 with ValidCompactorTxnList

use of org.apache.hadoop.hive.common.ValidCompactorTxnList in project hive by apache.

the class TestAcidUtils method deltasWithOpenTxnsNotInCompact.

@Test
public void deltasWithOpenTxnsNotInCompact() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("4:" + Long.MAX_VALUE));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(1, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
}

Example 2 with ValidCompactorTxnList

use of org.apache.hadoop.hive.common.ValidCompactorTxnList in project hive by apache.

the class TestAcidUtils method deltasAndDeleteDeltasWithOpenTxnsNotInCompact.

@Test
public void deltasAndDeleteDeltasWithOpenTxnsNotInCompact() throws Exception {
    // This tests checks that appropriate delta and delete_deltas are included when minor
    // compactions specifies a valid open txn range.
    Configuration conf = new Configuration();
    conf.setInt(HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES.varname, AcidUtils.AcidOperationalProperties.getDefault().toInt());
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_2_2/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0" + AcidUtils.DELTA_SIDE_FILE_SUFFIX, 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_7_7/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("4:" + Long.MAX_VALUE + ":"));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(2, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/delete_delta_2_2", delts.get(1).getPath().toString());
}

Example 3 with ValidCompactorTxnList

use of org.apache.hadoop.hive.common.ValidCompactorTxnList in project hive by apache.

the class TxnUtils method createValidCompactTxnList.

/**
   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse} to a
   * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
   * compact the files, and thus treats only open transactions as invalid.  Additionally any
   * txnId > highestOpenTxnId is also invalid.  This is to avoid creating something like
   * delta_17_120 where txnId 80, for example, is still open.
   * @param txns txn list from the metastore
   * @return a valid txn list.
   */
public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
    long highWater = txns.getTxn_high_water_mark();
    long minOpenTxn = Long.MAX_VALUE;
    long[] exceptions = new long[txns.getOpen_txnsSize()];
    int i = 0;
    for (TxnInfo txn : txns.getOpen_txns()) {
        if (txn.getState() == TxnState.OPEN) {
            minOpenTxn = Math.min(minOpenTxn, txn.getId());
        } else {
            //only need aborted since we don't consider anything above minOpenTxn
            exceptions[i++] = txn.getId();
        }
    }
    if (i < exceptions.length) {
        exceptions = Arrays.copyOf(exceptions, i);
    }
    highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
    return new ValidCompactorTxnList(exceptions, highWater);
}

Example 4 with ValidCompactorTxnList

use of org.apache.hadoop.hive.common.ValidCompactorTxnList in project hive by apache.

the class TestValidCompactorTxnList method maxTxnLowNoExceptions.

@Test
public void maxTxnLowNoExceptions() {
    ValidTxnList txns = new ValidCompactorTxnList(new long[0], 15);
    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
}

Example 5 with ValidCompactorTxnList

use of org.apache.hadoop.hive.common.ValidCompactorTxnList in project hive by apache.

the class TestValidCompactorTxnList method minTxnHighNoExceptions.

@Test
public void minTxnHighNoExceptions() {
    ValidTxnList txns = new ValidCompactorTxnList(new long[0], 5);
    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
}