Examples with SEQUENCEFILE - io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE

Example 1 with SEQUENCEFILE

use of io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE in project hetu-core by openlookeng.

the class TestHiveFileFormats method testSequenceFile.

@Test(dataProvider = "rowCount")
public void testSequenceFile(int rowCount) throws Exception {
    List<TestColumn> testColumns = TEST_COLUMNS.stream().filter(column -> !column.getName().equals("t_map_null_key_complex_key_value")).collect(toList());
    assertThatFileFormat(SEQUENCEFILE).withColumns(testColumns).withRowsCount(rowCount).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}

Also used : Iterables.transform(com.google.common.collect.Iterables.transform) RCBINARY(io.prestosql.plugin.hive.HiveStorageFormat.RCBINARY) Test(org.testng.annotations.Test) TEXTFILE(io.prestosql.plugin.hive.HiveStorageFormat.TEXTFILE) FileSplit(org.apache.hadoop.mapred.FileSplit) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Predicates.not(com.google.common.base.Predicates.not) Locale(java.util.Locale) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Duration(java.time.Duration) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) JSON(io.prestosql.plugin.hive.HiveStorageFormat.JSON) SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB) LzoCodec(io.airlift.compress.lzo.LzoCodec) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) BeforeClass(org.testng.annotations.BeforeClass) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assert.assertNotNull(org.testng.Assert.assertNotNull) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) RCTEXT(io.prestosql.plugin.hive.HiveStorageFormat.RCTEXT) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) ParquetPageSourceFactory(io.prestosql.plugin.hive.parquet.ParquetPageSourceFactory) HDFS_ENVIRONMENT(io.prestosql.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT) Optional(java.util.Optional) Iterables.filter(com.google.common.collect.Iterables.filter) ORC(io.prestosql.plugin.hive.HiveStorageFormat.ORC) Joiner(com.google.common.base.Joiner) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) DataProvider(org.testng.annotations.DataProvider) Logger(io.airlift.log.Logger) PrimitiveCategory(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) Assert.assertEquals(org.testng.Assert.assertEquals) RcFilePageSourceFactory(io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory) OptionalInt(java.util.OptionalInt) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector) LzopCodec(io.airlift.compress.lzo.LzopCodec) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcPageSourceFactory(io.prestosql.plugin.hive.orc.OrcPageSourceFactory) AVRO(io.prestosql.plugin.hive.HiveStorageFormat.AVRO) SEQUENCEFILE(io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE) RecordCursor(io.prestosql.spi.connector.RecordCursor) PARQUET(io.prestosql.plugin.hive.HiveStorageFormat.PARQUET) Properties(java.util.Properties) TupleDomain(io.prestosql.spi.predicate.TupleDomain) CSV(io.prestosql.plugin.hive.HiveStorageFormat.CSV) Assert.fail(org.testng.Assert.fail) IOException(java.io.IOException) File(java.io.File) OrcWriterOptions(io.prestosql.orc.OrcWriterOptions) HiveTestUtils.createGenericHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.createGenericHiveRecordCursorProvider) Collectors.toList(java.util.stream.Collectors.toList) OrcCacheStore(io.prestosql.orc.OrcCacheStore) FILE_INPUT_FORMAT(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) RecordPageSource(io.prestosql.spi.connector.RecordPageSource) HiveTestUtils.createGenericHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.createGenericHiveRecordCursorProvider) Test(org.testng.annotations.Test)

Example 2 with SEQUENCEFILE

use of io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE in project boostkit-bigdata by kunpengcompute.

the class TestHiveFileFormats method testSequenceFile.

@Test(dataProvider = "rowCount")
public void testSequenceFile(int rowCount) throws Exception {
    List<TestColumn> testColumns = TEST_COLUMNS.stream().filter(column -> !column.getName().equals("t_map_null_key_complex_key_value")).collect(toList());
    assertThatFileFormat(SEQUENCEFILE).withColumns(testColumns).withRowsCount(rowCount).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}

Aggregations

Joiner (com.google.common.base.Joiner)2 Predicates.not (com.google.common.base.Predicates.not)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Iterables.filter (com.google.common.collect.Iterables.filter)2 Iterables.transform (com.google.common.collect.Iterables.transform)2 Lists (com.google.common.collect.Lists)2 LzoCodec (io.airlift.compress.lzo.LzoCodec)2 LzopCodec (io.airlift.compress.lzo.LzopCodec)2 Logger (io.airlift.log.Logger)2 Slices (io.airlift.slice.Slices)2 Slices.utf8Slice (io.airlift.slice.Slices.utf8Slice)2 OrcCacheStore (io.prestosql.orc.OrcCacheStore)2 OrcWriterOptions (io.prestosql.orc.OrcWriterOptions)2 AVRO (io.prestosql.plugin.hive.HiveStorageFormat.AVRO)2 CSV (io.prestosql.plugin.hive.HiveStorageFormat.CSV)2 JSON (io.prestosql.plugin.hive.HiveStorageFormat.JSON)2 ORC (io.prestosql.plugin.hive.HiveStorageFormat.ORC)2