Examples with MockPath org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath used on opensource projects

Example 11 with MockPath

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath in project hive by apache.

the class TestAcidUtils method deltasWithOpenTxnsNotInCompact2.

@Test
public void deltasWithOpenTxnsNotInCompact2() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0" + AcidUtils.DELTA_SIDE_FILE_SUFFIX, 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("3:" + Long.MAX_VALUE));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(1, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
}

Example 12 with MockPath

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath in project hive by apache.

the class TestAcidUtils method testBestBase.

@Test
public void testBestBase() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_98_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_120_130/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_200/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    List<FileStatus> obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("10:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString());
    assertEquals(0, dir.getCurrentDirectories().size());
    obsoletes = dir.getObsolete();
    assertEquals(1, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(0).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    /*Single statemnt txns only: since we don't compact a txn range that includes an open txn,
    the existence of delta_120_130 implies that 121 in the exception list is aborted unless
    delta_120_130 is from streaming ingest in which case 121 can be open
    (and thus 122-130 are open too)
    99 here would be Aborted since 121 is minOpenTxn, base_100 is still good
    For multi-statment txns, see HIVE-13369*/
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:121:99:121"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    boolean gotException = false;
    try {
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("125:5:5"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (125,5).  Oldest available base: " + "mock:/tbl/part1/base_5", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_12_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_2_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, //non-acid to acid table conversion
    new MockFile("mock:/tbl/part1/base_" + Long.MIN_VALUE + "/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    //note that we don't include current txn of the client in exception list to read-you-writes
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("1:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_" + Long.MIN_VALUE, dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_1_1", dir.getCurrentDirectories().get(0).getPath().toString());
    assertEquals(0, dir.getObsolete().size());
}

Example 13 with MockPath

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath in project hive by apache.

the class TestAcidUtils method deleteDeltasWithOpenTxnInRead.

@Test
public void deleteDeltasWithOpenTxnInRead() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES.varname, AcidUtils.AcidOperationalProperties.getDefault().toInt());
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_3_3/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_4_4_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_4_4_3/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_101_101_1/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:4:4"));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(3, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/delete_delta_2_5", delts.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_2_5", delts.get(2).getPath().toString());
// Note that delete_delta_3_3 should not be read, when a minor compacted
// [delete_]delta_2_5 is present.
}