Examples with ReaderPair org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.ReaderPair used on opensource projects

Example 1 with ReaderPair

use of org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.ReaderPair in project hive by apache.

the class TestOrcRawRecordMerger method testOriginalReaderPair.

@Test
public void testOriginalReaderPair() throws Exception {
    int BUCKET = 10;
    ReaderKey key = new ReaderKey();
    Configuration conf = new Configuration();
    int bucketProperty = OrcRawRecordMerger.encodeBucketId(conf, BUCKET, 0);
    Reader reader = createMockOriginalReader();
    RecordIdentifier minKey = new RecordIdentifier(0, bucketProperty, 1);
    RecordIdentifier maxKey = new RecordIdentifier(0, bucketProperty, 3);
    boolean[] includes = new boolean[] { true, true };
    FileSystem fs = FileSystem.getLocal(conf);
    Path root = new Path(tmpDir, "testOriginalReaderPair");
    fs.makeQualified(root);
    fs.create(root);
    ReaderPair pair = new OrcRawRecordMerger.OriginalReaderPairToRead(key, reader, BUCKET, minKey, maxKey, new Reader.Options().include(includes), new OrcRawRecordMerger.Options().rootPath(root), conf, new ValidReaderWriteIdList(), 0);
    RecordReader recordReader = pair.getRecordReader();
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(2, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    assertEquals("third", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(3, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    assertEquals("fourth", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(null, pair.nextRecord());
    Mockito.verify(recordReader).close();
}

Example 2 with ReaderPair

use of org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.ReaderPair in project hive by apache.

the class TestOrcRawRecordMerger method testReaderPairNoMin.

@Test
public void testReaderPairNoMin() throws Exception {
    ReaderKey key = new ReaderKey();
    Reader reader = createMockReader();
    ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, null, null, new Reader.Options(), new HiveConf());
    RecordReader recordReader = pair.getRecordReader();
    assertEquals(10, key.getWriteId());
    assertEquals(20, key.getBucketProperty());
    assertEquals(20, key.getRowId());
    assertEquals(100, key.getCurrentWriteId());
    assertEquals("first", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(10, key.getWriteId());
    assertEquals(20, key.getBucketProperty());
    assertEquals(30, key.getRowId());
    assertEquals(110, key.getCurrentWriteId());
    assertEquals("second", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(10, key.getWriteId());
    assertEquals(20, key.getBucketProperty());
    assertEquals(40, key.getRowId());
    assertEquals(120, key.getCurrentWriteId());
    assertEquals("third", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(40, key.getWriteId());
    assertEquals(50, key.getBucketProperty());
    assertEquals(60, key.getRowId());
    assertEquals(130, key.getCurrentWriteId());
    assertEquals("fourth", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(40, key.getWriteId());
    assertEquals(50, key.getBucketProperty());
    assertEquals(61, key.getRowId());
    assertEquals(140, key.getCurrentWriteId());
    assertEquals("fifth", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(null, pair.nextRecord());
    Mockito.verify(recordReader).close();
}

Example 3 with ReaderPair

use of org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.ReaderPair in project hive by apache.

the class TestOrcRawRecordMerger method testOriginalReaderPairNoMin.

@Test
public void testOriginalReaderPairNoMin() throws Exception {
    int BUCKET = 10;
    ReaderKey key = new ReaderKey();
    Reader reader = createMockOriginalReader();
    Configuration conf = new Configuration();
    int bucketProperty = OrcRawRecordMerger.encodeBucketId(conf, BUCKET, 0);
    FileSystem fs = FileSystem.getLocal(conf);
    Path root = new Path(tmpDir, "testOriginalReaderPairNoMin");
    fs.makeQualified(root);
    fs.create(root);
    ReaderPair pair = new OrcRawRecordMerger.OriginalReaderPairToRead(key, reader, BUCKET, null, null, new Reader.Options(), new OrcRawRecordMerger.Options().rootPath(root), conf, new ValidReaderWriteIdList(), 0);
    assertEquals("first", value(pair.nextRecord()));
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(0, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    pair.next(pair.nextRecord());
    assertEquals("second", value(pair.nextRecord()));
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(1, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    pair.next(pair.nextRecord());
    assertEquals("third", value(pair.nextRecord()));
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(2, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    pair.next(pair.nextRecord());
    assertEquals("fourth", value(pair.nextRecord()));
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(3, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    pair.next(pair.nextRecord());
    assertEquals("fifth", value(pair.nextRecord()));
    assertEquals(0, key.getWriteId());
    assertEquals(bucketProperty, key.getBucketProperty());
    assertEquals(4, key.getRowId());
    assertEquals(0, key.getCurrentWriteId());
    pair.next(pair.nextRecord());
    assertEquals(null, pair.nextRecord());
    Mockito.verify(pair.getRecordReader()).close();
}

Example 4 with ReaderPair

use of org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.ReaderPair in project hive by apache.

the class TestOrcRawRecordMerger method testReaderPair.

@Test
public void testReaderPair() throws Exception {
    ReaderKey key = new ReaderKey();
    Reader reader = createMockReader();
    RecordIdentifier minKey = new RecordIdentifier(10, 20, 30);
    RecordIdentifier maxKey = new RecordIdentifier(40, 50, 60);
    ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, minKey, maxKey, new Reader.Options(), new HiveConf());
    RecordReader recordReader = pair.getRecordReader();
    assertEquals(10, key.getWriteId());
    assertEquals(20, key.getBucketProperty());
    assertEquals(40, key.getRowId());
    assertEquals(120, key.getCurrentWriteId());
    assertEquals("third", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(40, key.getWriteId());
    assertEquals(50, key.getBucketProperty());
    assertEquals(60, key.getRowId());
    assertEquals(130, key.getCurrentWriteId());
    assertEquals("fourth", value(pair.nextRecord()));
    pair.next(pair.nextRecord());
    assertEquals(null, pair.nextRecord());
    Mockito.verify(recordReader).close();
}