Example 1 with DwrfPageSourceFactory
use of com.facebook.presto.hive.orc.DwrfPageSourceFactory in project presto by prestodb.
the class TestHiveFileFormats method testDwrf.
@Test(dataProvider = "rowCount")
public void testDwrf(int rowCount) throws Exception {
List<TestColumn> testColumns = TEST_COLUMNS.stream().filter(testColumn -> !hasType(testColumn.getObjectInspector(), PrimitiveCategory.DATE, PrimitiveCategory.VARCHAR, PrimitiveCategory.CHAR, PrimitiveCategory.DECIMAL)).collect(Collectors.toList());
assertThatFileFormat(DWRF).withColumns(testColumns).withRowsCount(rowCount).isReadableByPageSource(new DwrfPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT));
}
Also used :
RecordPageSource(com.facebook.presto.spi.RecordPageSource)
DateTimeZone(org.joda.time.DateTimeZone)
ORC(com.facebook.presto.hive.HiveStorageFormat.ORC)
Iterables.transform(com.google.common.collect.Iterables.transform)
OrcPageSourceFactory(com.facebook.presto.hive.orc.OrcPageSourceFactory)
Test(org.testng.annotations.Test)
RowType(com.facebook.presto.type.RowType)
FileSplit(org.apache.hadoop.mapred.FileSplit)
Predicates.not(com.google.common.base.Predicates.not)
Configuration(org.apache.hadoop.conf.Configuration)
AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO)
Path(org.apache.hadoop.fs.Path)
Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice)
ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)
ImmutableCollectors.toImmutableList(com.facebook.presto.util.ImmutableCollectors.toImmutableList)
TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE)
SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB)
ImmutableSet(com.google.common.collect.ImmutableSet)
ImmutableMap(com.google.common.collect.ImmutableMap)
TimeZone(java.util.TimeZone)
MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector)
BeforeClass(org.testng.annotations.BeforeClass)
DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF)
StructuralTestUtil.rowBlockOf(com.facebook.presto.tests.StructuralTestUtil.rowBlockOf)
Assert.assertNotNull(org.testng.Assert.assertNotNull)
StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)
Collectors(java.util.stream.Collectors)
ConnectorSession(com.facebook.presto.spi.ConnectorSession)
TupleDomain(com.facebook.presto.spi.predicate.TupleDomain)
RecordCursor(com.facebook.presto.spi.RecordCursor)
List(java.util.List)
StructuralTestUtil.arrayBlockOf(com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf)
VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo)
TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER)
Optional(java.util.Optional)
INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER)
Iterables.filter(com.google.common.collect.Iterables.filter)
PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector)
Joiner(com.google.common.base.Joiner)
StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField)
ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector)
DataProvider(org.testng.annotations.DataProvider)
PrimitiveCategory(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory)
ArrayType(com.facebook.presto.type.ArrayType)
HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes)
RcFilePageSourceFactory(com.facebook.presto.hive.rcfile.RcFilePageSourceFactory)
Assert.assertEquals(org.testng.Assert.assertEquals)
PrestoException(com.facebook.presto.spi.PrestoException)
OptionalInt(java.util.OptionalInt)
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector)
PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET)
HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar)
Lists(com.google.common.collect.Lists)
ImmutableList(com.google.common.collect.ImmutableList)
PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector)
SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION)
DwrfPageSourceFactory(com.facebook.presto.hive.orc.DwrfPageSourceFactory)
RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT)
Objects.requireNonNull(java.util.Objects.requireNonNull)
ParquetRecordCursorProvider(com.facebook.presto.hive.parquet.ParquetRecordCursorProvider)
JSON(com.facebook.presto.hive.HiveStorageFormat.JSON)
SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE)
Properties(java.util.Properties)
Assert.fail(org.testng.Assert.fail)
IOException(java.io.IOException)
ObjectInspectorFactory.getStandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector)
TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession)
ObjectInspectorFactory.getStandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector)
File(java.io.File)
RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY)
VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType)
Collectors.toList(java.util.stream.Collectors.toList)
ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource)
HDFS_ENVIRONMENT(com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT)
FILE_INPUT_FORMAT(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT)
Assert.assertTrue(org.testng.Assert.assertTrue)
PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector)
ParquetPageSourceFactory(com.facebook.presto.hive.parquet.ParquetPageSourceFactory)
DwrfPageSourceFactory(com.facebook.presto.hive.orc.DwrfPageSourceFactory)
Test(org.testng.annotations.Test)
Aggregations
AVRO (com.facebook.presto.hive.HiveStorageFormat.AVRO)1
DWRF (com.facebook.presto.hive.HiveStorageFormat.DWRF)1
JSON (com.facebook.presto.hive.HiveStorageFormat.JSON)1
ORC (com.facebook.presto.hive.HiveStorageFormat.ORC)1
PARQUET (com.facebook.presto.hive.HiveStorageFormat.PARQUET)1
RCBINARY (com.facebook.presto.hive.HiveStorageFormat.RCBINARY)1
RCTEXT (com.facebook.presto.hive.HiveStorageFormat.RCTEXT)1
SEQUENCEFILE (com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE)1
TEXTFILE (com.facebook.presto.hive.HiveStorageFormat.TEXTFILE)1
HDFS_ENVIRONMENT (com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT)1
SESSION (com.facebook.presto.hive.HiveTestUtils.SESSION)1
TYPE_MANAGER (com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER)1
HiveTestUtils.getTypes (com.facebook.presto.hive.HiveTestUtils.getTypes)1
DwrfPageSourceFactory (com.facebook.presto.hive.orc.DwrfPageSourceFactory)1
OrcPageSourceFactory (com.facebook.presto.hive.orc.OrcPageSourceFactory)1
ParquetPageSourceFactory (com.facebook.presto.hive.parquet.ParquetPageSourceFactory)1
ParquetRecordCursorProvider (com.facebook.presto.hive.parquet.ParquetRecordCursorProvider)1
RcFilePageSourceFactory (com.facebook.presto.hive.rcfile.RcFilePageSourceFactory)1
ConnectorPageSource (com.facebook.presto.spi.ConnectorPageSource)1
ConnectorSession (com.facebook.presto.spi.ConnectorSession)1
