Examples with OrcFileMetadata org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata used on opensource projects

Example 1 with OrcFileMetadata

use of org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata in project hive by apache.

the class TestIncrementalObjectSizeEstimator method testMetadata.

@Test
public void testMetadata() throws IOException {
    // Mostly tests that it doesn't crash.
    OrcStripeMetadata osm = OrcStripeMetadata.createDummy(0);
    HashMap<Class<?>, ObjectEstimator> map = IncrementalObjectSizeEstimator.createEstimators(osm);
    IncrementalObjectSizeEstimator.addEstimator("com.google.protobuf.LiteralByteString", map);
    ObjectEstimator root = map.get(OrcStripeMetadata.class);
    LOG.info("Estimated " + root.estimate(osm, map) + " for a dummy OSM");
    OrcBatchKey stripeKey = null;
    DummyMetadataReader mr = new DummyMetadataReader();
    mr.doStreamStep = false;
    mr.isEmpty = true;
    StripeInformation si = Mockito.mock(StripeInformation.class);
    Mockito.when(si.getNumberOfRows()).thenReturn(0L);
    osm = new OrcStripeMetadata(stripeKey, mr, si, null, null, null, null);
    LOG.info("Estimated " + root.estimate(osm, map) + " for an empty OSM");
    mr.doStreamStep = true;
    osm = new OrcStripeMetadata(stripeKey, mr, si, null, null, null, null);
    LOG.info("Estimated " + root.estimate(osm, map) + " for an empty OSM after serde");
    mr.isEmpty = false;
    stripeKey = new OrcBatchKey(0, 0, 0);
    osm = new OrcStripeMetadata(stripeKey, mr, si, null, null, null, null);
    LOG.info("Estimated " + root.estimate(osm, map) + " for a test OSM");
    osm.resetRowIndex();
    LOG.info("Estimated " + root.estimate(osm, map) + " for a test OSM w/o row index");
    mr.doStreamStep = true;
    osm = new OrcStripeMetadata(stripeKey, mr, si, null, null, null, null);
    LOG.info("Estimated " + root.estimate(osm, map) + " for a test OSM after serde");
    osm.resetRowIndex();
    LOG.info("Estimated " + root.estimate(osm, map) + " for a test OSM w/o row index after serde");
    OrcFileMetadata ofm = OrcFileMetadata.createDummy(0);
    map = IncrementalObjectSizeEstimator.createEstimators(ofm);
    IncrementalObjectSizeEstimator.addEstimator("com.google.protobuf.LiteralByteString", map);
    root = map.get(OrcFileMetadata.class);
    LOG.info("Estimated " + root.estimate(ofm, map) + " for a dummy OFM");
}

Example 2 with OrcFileMetadata

use of org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata in project hive by apache.

the class OrcEncodedDataReader method getOrReadFileMetadata.

/**
   *  Gets file metadata for the split from cache, or reads it from the file.
   */
private OrcFileMetadata getOrReadFileMetadata() throws IOException {
    OrcFileMetadata metadata = null;
    if (fileKey != null && metadataCache != null) {
        metadata = metadataCache.getFileMetadata(fileKey);
        if (metadata != null) {
            counters.incrCounter(LlapIOCounters.METADATA_CACHE_HIT);
            return metadata;
        } else {
            counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
        }
    }
    ensureOrcReader();
    // We assume this call doesn't touch HDFS because everything is already read; don't add time.
    metadata = new OrcFileMetadata(fileKey, orcReader);
    if (fileKey == null || metadataCache == null)
        return metadata;
    return metadataCache.putFileMetadata(metadata);
}

Example 3 with OrcFileMetadata

use of org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata in project hive by apache.

the class TestOrcMetadataCache method testGetPut.

@Test
public void testGetPut() throws Exception {
    DummyMemoryManager mm = new DummyMemoryManager();
    DummyCachePolicy cp = new DummyCachePolicy();
    OrcMetadataCache cache = new OrcMetadataCache(mm, cp, false);
    OrcFileMetadata ofm1 = OrcFileMetadata.createDummy(1), ofm2 = OrcFileMetadata.createDummy(2);
    assertSame(ofm1, cache.putFileMetadata(ofm1));
    assertEquals(1, mm.allocs);
    cp.verifyEquals(1);
    assertSame(ofm2, cache.putFileMetadata(ofm2));
    assertEquals(2, mm.allocs);
    cp.verifyEquals(2);
    assertSame(ofm1, cache.getFileMetadata(1));
    assertSame(ofm2, cache.getFileMetadata(2));
    cp.verifyEquals(4);
    OrcFileMetadata ofm3 = OrcFileMetadata.createDummy(1);
    assertSame(ofm1, cache.putFileMetadata(ofm3));
    assertEquals(2, mm.allocs);
    cp.verifyEquals(5);
    assertSame(ofm1, cache.getFileMetadata(1));
    cp.verifyEquals(6);
    OrcStripeMetadata osm1 = OrcStripeMetadata.createDummy(1), osm2 = OrcStripeMetadata.createDummy(2);
    assertSame(osm1, cache.putStripeMetadata(osm1));
    assertEquals(3, mm.allocs);
    assertSame(osm2, cache.putStripeMetadata(osm2));
    assertEquals(4, mm.allocs);
    assertSame(osm1, cache.getStripeMetadata(osm1.getKey()));
    assertSame(osm2, cache.getStripeMetadata(osm2.getKey()));
    OrcStripeMetadata osm3 = OrcStripeMetadata.createDummy(1);
    assertSame(osm1, cache.putStripeMetadata(osm3));
    assertEquals(4, mm.allocs);
    assertSame(osm1, cache.getStripeMetadata(osm3.getKey()));
    cp.verifyEquals(12);
}