Examples with MockFileSystem org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem used on opensource projects

Example 11 with MockFileSystem

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem in project hive by apache.

the class TestAcidUtils method deltasAndDeleteDeltasWithOpenTxnsNotInCompact.

@Test
public void deltasAndDeleteDeltasWithOpenTxnsNotInCompact() throws Exception {
    // This tests checks that appropriate delta and delete_deltas are included when minor
    // compactions specifies a valid open txn range.
    Configuration conf = new Configuration();
    conf.setInt(HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES.varname, AcidUtils.AcidOperationalProperties.getDefault().toInt());
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_2_2/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0" + AcidUtils.DELTA_SIDE_FILE_SUFFIX, 500, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_7_7/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("4:" + Long.MAX_VALUE + ":"));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(2, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/delete_delta_2_2", delts.get(1).getPath().toString());
}

Example 12 with MockFileSystem

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem in project hive by apache.

the class TestAcidUtils method testBaseWithDeleteDeltas.

@Test
public void testBaseWithDeleteDeltas() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES.varname, AcidOperationalProperties.getDefault().toInt());
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_029_029/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_025_030/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_025_030/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_050_105/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_050_105/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delete_delta_110_110/bucket_0", 0, new byte[0]));
    AcidUtils.Directory dir = AcidUtils.getAcidState(new TestInputOutputFormat.MockPath(fs, "mock:/tbl/part1"), conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_49", dir.getBaseDirectory().toString());
    List<FileStatus> obsolete = dir.getObsolete();
    assertEquals(7, obsolete.size());
    assertEquals("mock:/tbl/part1/base_10", obsolete.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsolete.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/delete_delta_025_030", obsolete.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_025_030", obsolete.get(3).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_025_025", obsolete.get(4).getPath().toString());
    assertEquals("mock:/tbl/part1/delete_delta_029_029", obsolete.get(5).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_029_029", obsolete.get(6).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    List<AcidUtils.ParsedDelta> deltas = dir.getCurrentDirectories();
    assertEquals(2, deltas.size());
    assertEquals("mock:/tbl/part1/delete_delta_050_105", deltas.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_050_105", deltas.get(1).getPath().toString());
// The delete_delta_110_110 should not be read because it is greater than the high watermark.
}

Example 13 with MockFileSystem

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem in project hive by apache.

the class TestAcidUtils method deltasWithOpenTxnsNotInCompact2.

@Test
public void deltasWithOpenTxnsNotInCompact2() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0" + AcidUtils.DELTA_SIDE_FILE_SUFFIX, 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "mock:/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("3:" + Long.MAX_VALUE));
    List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories();
    assertEquals(1, delts.size());
    assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString());
}

Example 14 with MockFileSystem

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem in project hive by apache.

the class TestAcidUtils method testBestBase.

@Test
public void testBestBase() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_98_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_120_130/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_200/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    List<FileStatus> obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("10:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString());
    assertEquals(0, dir.getCurrentDirectories().size());
    obsoletes = dir.getObsolete();
    assertEquals(1, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(0).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    /*Single statemnt txns only: since we don't compact a txn range that includes an open txn,
    the existence of delta_120_130 implies that 121 in the exception list is aborted unless
    delta_120_130 is from streaming ingest in which case 121 can be open
    (and thus 122-130 are open too)
    99 here would be Aborted since 121 is minOpenTxn, base_100 is still good
    For multi-statment txns, see HIVE-13369*/
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:121:99:121"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    boolean gotException = false;
    try {
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("125:5:5"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (125,5).  Oldest available base: " + "mock:/tbl/part1/base_5", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_12_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_2_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, //non-acid to acid table conversion
    new MockFile("mock:/tbl/part1/base_" + Long.MIN_VALUE + "/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    //note that we don't include current txn of the client in exception list to read-you-writes
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("1:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_" + Long.MIN_VALUE, dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_1_1", dir.getCurrentDirectories().get(0).getPath().toString());
    assertEquals(0, dir.getObsolete().size());
}

Example 15 with MockFileSystem

use of org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem in project hive by apache.

the class TestAcidUtils method testBaseDeltas.

@Test
public void testBaseDeltas() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_49/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_025_025/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_029_029/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_025_030/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_050_105/bucket_0", 0, new byte[0]), new MockFile("mock:/tbl/part1/delta_90_120/bucket_0", 0, new byte[0]));
    AcidUtils.Directory dir = AcidUtils.getAcidState(new TestInputOutputFormat.MockPath(fs, "mock:/tbl/part1"), conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_49", dir.getBaseDirectory().toString());
    List<FileStatus> obsolete = dir.getObsolete();
    assertEquals(5, obsolete.size());
    assertEquals("mock:/tbl/part1/base_10", obsolete.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsolete.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_025_030", obsolete.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_025_025", obsolete.get(3).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_029_029", obsolete.get(4).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    List<AcidUtils.ParsedDelta> deltas = dir.getCurrentDirectories();
    assertEquals(1, deltas.size());
    AcidUtils.ParsedDelta delt = deltas.get(0);
    assertEquals("mock:/tbl/part1/delta_050_105", delt.getPath().toString());
    assertEquals(50, delt.getMinTransaction());
    assertEquals(105, delt.getMaxTransaction());
}