Search in sources :

Example 1 with TYPE_MANAGER

use of io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER in project hetu-core by openlookeng.

the class AbstractTestHiveFileFormats method checkCursor.

protected void checkCursor(RecordCursor cursor, List<TestColumn> testColumns, int rowCount) {
    List<Type> types = testColumns.stream().map(column -> column.getObjectInspector().getTypeName()).map(type -> HiveType.valueOf(type).getType(TYPE_MANAGER)).collect(toImmutableList());
    Map<Type, MethodHandle> distinctFromOperators = types.stream().distinct().collect(toImmutableMap(identity(), HiveTestUtils::distinctFromOperator));
    for (int row = 0; row < rowCount; row++) {
        assertTrue(cursor.advanceNextPosition());
        for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
            TestColumn testColumn = testColumns.get(i);
            Type type = types.get(i);
            Object fieldFromCursor = getFieldFromCursor(cursor, type, i);
            if (fieldFromCursor == null) {
                assertEquals(null, testColumn.getExpectedValue(), "Expected null for column " + testColumn.getName());
            } else if (type instanceof DecimalType) {
                DecimalType decimalType = (DecimalType) type;
                fieldFromCursor = new BigDecimal((BigInteger) fieldFromCursor, decimalType.getScale());
                assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
            } else if (testColumn.getObjectInspector().getTypeName().equals("float")) {
                assertEquals((float) fieldFromCursor, (float) testColumn.getExpectedValue(), (float) EPSILON);
            } else if (testColumn.getObjectInspector().getTypeName().equals("double")) {
                assertEquals((double) fieldFromCursor, (double) testColumn.getExpectedValue(), EPSILON);
            } else if (testColumn.getObjectInspector().getTypeName().equals("tinyint")) {
                assertEquals(((Number) fieldFromCursor).byteValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getTypeName().equals("smallint")) {
                assertEquals(((Number) fieldFromCursor).shortValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getTypeName().equals("int")) {
                assertEquals(((Number) fieldFromCursor).intValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
                assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
            } else {
                Block expected = (Block) testColumn.getExpectedValue();
                Block actual = (Block) fieldFromCursor;
                boolean distinct = isDistinctFrom(distinctFromOperators.get(type), expected, actual);
                assertFalse(distinct, "Wrong value for column: " + testColumn.getName());
            }
        }
    }
    assertFalse(cursor.advanceNextPosition());
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) StorageFormat(io.prestosql.plugin.hive.metastore.StorageFormat) JavaHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory.javaByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteObjectInspector) Text(org.apache.hadoop.io.Text) PrimitiveObjectInspectorFactory.javaLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector) Writable(org.apache.hadoop.io.Writable) Test(org.testng.annotations.Test) PrimitiveObjectInspectorFactory.javaTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector) Date(org.apache.hadoop.hive.common.type.Date) HiveTestUtils.isDistinctFrom(io.prestosql.plugin.hive.HiveTestUtils.isDistinctFrom) PrimitiveObjectInspectorFactory.javaDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDateObjectInspector) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) BigDecimal(java.math.BigDecimal) FileSplit(org.apache.hadoop.mapred.FileSplit) Predicates.not(com.google.common.base.Predicates.not) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) RowType(io.prestosql.spi.type.RowType) BigInteger(java.math.BigInteger) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) Assert.assertFalse(org.testng.Assert.assertFalse) Chars.isCharType(io.prestosql.spi.type.Chars.isCharType) PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector) PrimitiveObjectInspectorFactory.javaFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaFloatObjectInspector) PrimitiveObjectInspectorFactory.javaDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDoubleObjectInspector) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) TimestampType(io.prestosql.spi.type.TimestampType) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) StructuralTestUtil.mapBlockOf(io.prestosql.tests.StructuralTestUtil.mapBlockOf) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Iterables.filter(com.google.common.collect.Iterables.filter) StructuralTestUtil.decimalMapBlockOf(io.prestosql.tests.StructuralTestUtil.decimalMapBlockOf) PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector) Joiner(com.google.common.base.Joiner) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) JavaHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveCharObjectInspector) MethodHandle(java.lang.invoke.MethodHandle) Slice(io.airlift.slice.Slice) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) StructuralTestUtil.decimalArrayBlockOf(io.prestosql.tests.StructuralTestUtil.decimalArrayBlockOf) CharType(io.prestosql.spi.type.CharType) StructuralTestUtil.rowBlockOf(io.prestosql.tests.StructuralTestUtil.rowBlockOf) ArrayList(java.util.ArrayList) REGULAR(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR) PrimitiveObjectInspectorFactory.javaShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaShortObjectInspector) PARTITION_KEY(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) RecordCursor(io.prestosql.spi.connector.RecordCursor) StructuralTestUtil(io.prestosql.tests.StructuralTestUtil) SESSION(io.prestosql.plugin.hive.HiveTestUtils.SESSION) HiveTestUtils.mapType(io.prestosql.plugin.hive.HiveTestUtils.mapType) Arrays.fill(java.util.Arrays.fill) Properties(java.util.Properties) IOException(java.io.IOException) ObjectInspectorFactory.getStandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector) UTC(org.joda.time.DateTimeZone.UTC) MaterializedRow(io.prestosql.testing.MaterializedRow) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) File(java.io.File) SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) VARBINARY(io.prestosql.spi.type.VarbinaryType.VARBINARY) DateTimeTestingUtils.sqlTimestampOf(io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_DEFAULT_DYNAMIC_PARTITION(io.prestosql.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION) RecordWriter(org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter) Varchars.isVarcharType(io.prestosql.spi.type.Varchars.isVarcharType) Iterables.transform(com.google.common.collect.Iterables.transform) DecimalType(io.prestosql.spi.type.DecimalType) MaterializedResult.materializeSourceDataStream(io.prestosql.testing.MaterializedResult.materializeSourceDataStream) MaterializedResult(io.prestosql.testing.MaterializedResult) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Path(org.apache.hadoop.fs.Path) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) Type(io.prestosql.spi.type.Type) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) DateTimeFormat(org.joda.time.format.DateTimeFormat) ImmutableMap(com.google.common.collect.ImmutableMap) BlockBuilder(io.prestosql.spi.block.BlockBuilder) ArrayType(io.prestosql.spi.type.ArrayType) CharType.createCharType(io.prestosql.spi.type.CharType.createCharType) TINYINT(io.prestosql.spi.type.TinyintType.TINYINT) HiveOutputFormat(org.apache.hadoop.hive.ql.io.HiveOutputFormat) PageBuilder(io.prestosql.spi.PageBuilder) List(java.util.List) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) TypeInfoFactory.getCharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) Strings.padEnd(com.google.common.base.Strings.padEnd) HiveUtil.isStructuralType(io.prestosql.plugin.hive.HiveUtil.isStructuralType) PrimitiveObjectInspectorFactory.javaBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector) SqlDate(io.prestosql.spi.type.SqlDate) SerDeUtils.serializeObject(io.prestosql.plugin.hive.util.SerDeUtils.serializeObject) Assert.assertEquals(org.testng.Assert.assertEquals) Decimals(io.prestosql.spi.type.Decimals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) SqlDecimal(io.prestosql.spi.type.SqlDecimal) HdfsConfigurationInitializer.configureCompression(io.prestosql.plugin.hive.HdfsConfigurationInitializer.configureCompression) Float.intBitsToFloat(java.lang.Float.intBitsToFloat) SqlVarbinary(io.prestosql.spi.type.SqlVarbinary) Category(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) REAL(io.prestosql.spi.type.RealType.REAL) ObjectInspectorFactory.getStandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector) Block(io.prestosql.spi.block.Block) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) UTF_8(java.nio.charset.StandardCharsets.UTF_8) DateTime(org.joda.time.DateTime) Page(io.prestosql.spi.Page) ObjectInspectorFactory.getStandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector) JobConf(org.apache.hadoop.mapred.JobConf) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) SMALLINT(io.prestosql.spi.type.SmallintType.SMALLINT) DateType(io.prestosql.spi.type.DateType) Serializer(org.apache.hadoop.hive.serde2.Serializer) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) StructuralTestUtil.arrayBlockOf(io.prestosql.tests.StructuralTestUtil.arrayBlockOf) Assert.assertTrue(org.testng.Assert.assertTrue) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) RowType(io.prestosql.spi.type.RowType) Chars.isCharType(io.prestosql.spi.type.Chars.isCharType) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) TimestampType(io.prestosql.spi.type.TimestampType) CharType(io.prestosql.spi.type.CharType) HiveTestUtils.mapType(io.prestosql.plugin.hive.HiveTestUtils.mapType) Varchars.isVarcharType(io.prestosql.spi.type.Varchars.isVarcharType) DecimalType(io.prestosql.spi.type.DecimalType) Type(io.prestosql.spi.type.Type) ArrayType(io.prestosql.spi.type.ArrayType) CharType.createCharType(io.prestosql.spi.type.CharType.createCharType) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) HiveUtil.isStructuralType(io.prestosql.plugin.hive.HiveUtil.isStructuralType) DateType(io.prestosql.spi.type.DateType) DecimalType(io.prestosql.spi.type.DecimalType) Block(io.prestosql.spi.block.Block) SerDeUtils.serializeObject(io.prestosql.plugin.hive.util.SerDeUtils.serializeObject) BigDecimal(java.math.BigDecimal) MethodHandle(java.lang.invoke.MethodHandle)

Example 2 with TYPE_MANAGER

use of io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER in project hetu-core by openlookeng.

the class TestHiveFileFormats method testPageSourceFactory.

private void testPageSourceFactory(HivePageSourceFactory sourceFactory, FileSplit split, HiveStorageFormat storageFormat, List<TestColumn> testColumns, ConnectorSession session, int rowCount) throws IOException {
    Properties splitProperties = new Properties();
    splitProperties.setProperty(FILE_INPUT_FORMAT, storageFormat.getInputFormat());
    splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe());
    splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName)));
    splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType)));
    List<HivePartitionKey> partitionKeys = testColumns.stream().filter(TestColumn::isPartitionKey).map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())).collect(toList());
    List<HiveColumnHandle> columnHandles = getColumnHandles(testColumns);
    Optional<ConnectorPageSource> pageSource = HivePageSourceProvider.createHivePageSource(ImmutableSet.of(), ImmutableSet.of(sourceFactory), new Configuration(), session, split.getPath(), OptionalInt.empty(), split.getStart(), split.getLength(), split.getLength(), splitProperties, TupleDomain.all(), columnHandles, partitionKeys, TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L, ImmutableMap.of(), ImmutableList.of());
    assertTrue(pageSource.isPresent());
    checkPageSource(pageSource.get(), testColumns, HiveTestUtils.getTypes(columnHandles), rowCount);
}
Also used : Iterables.transform(com.google.common.collect.Iterables.transform) RCBINARY(io.prestosql.plugin.hive.HiveStorageFormat.RCBINARY) Test(org.testng.annotations.Test) TEXTFILE(io.prestosql.plugin.hive.HiveStorageFormat.TEXTFILE) FileSplit(org.apache.hadoop.mapred.FileSplit) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Predicates.not(com.google.common.base.Predicates.not) Locale(java.util.Locale) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Duration(java.time.Duration) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) JSON(io.prestosql.plugin.hive.HiveStorageFormat.JSON) SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB) LzoCodec(io.airlift.compress.lzo.LzoCodec) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) BeforeClass(org.testng.annotations.BeforeClass) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assert.assertNotNull(org.testng.Assert.assertNotNull) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) RCTEXT(io.prestosql.plugin.hive.HiveStorageFormat.RCTEXT) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) ParquetPageSourceFactory(io.prestosql.plugin.hive.parquet.ParquetPageSourceFactory) HDFS_ENVIRONMENT(io.prestosql.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT) Optional(java.util.Optional) Iterables.filter(com.google.common.collect.Iterables.filter) ORC(io.prestosql.plugin.hive.HiveStorageFormat.ORC) Joiner(com.google.common.base.Joiner) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) DataProvider(org.testng.annotations.DataProvider) Logger(io.airlift.log.Logger) PrimitiveCategory(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) Assert.assertEquals(org.testng.Assert.assertEquals) RcFilePageSourceFactory(io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory) OptionalInt(java.util.OptionalInt) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector) LzopCodec(io.airlift.compress.lzo.LzopCodec) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcPageSourceFactory(io.prestosql.plugin.hive.orc.OrcPageSourceFactory) AVRO(io.prestosql.plugin.hive.HiveStorageFormat.AVRO) SEQUENCEFILE(io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE) RecordCursor(io.prestosql.spi.connector.RecordCursor) PARQUET(io.prestosql.plugin.hive.HiveStorageFormat.PARQUET) Properties(java.util.Properties) TupleDomain(io.prestosql.spi.predicate.TupleDomain) CSV(io.prestosql.plugin.hive.HiveStorageFormat.CSV) Assert.fail(org.testng.Assert.fail) IOException(java.io.IOException) File(java.io.File) OrcWriterOptions(io.prestosql.orc.OrcWriterOptions) HiveTestUtils.createGenericHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.createGenericHiveRecordCursorProvider) Collectors.toList(java.util.stream.Collectors.toList) OrcCacheStore(io.prestosql.orc.OrcCacheStore) FILE_INPUT_FORMAT(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) RecordPageSource(io.prestosql.spi.connector.RecordPageSource) Configuration(org.apache.hadoop.conf.Configuration) Properties(java.util.Properties) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource)

Example 3 with TYPE_MANAGER

use of io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, Function<HiveConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        config.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(config, host, port);
    ExecutorService executors = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    ExecutorService executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, config, new NoHdfsAuthentication());
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executors, executorRefresh, config, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    metadataFactory = new HiveMetadataFactory(config, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), vacuumExecutorService, heartbeatService, vacuumExecutorService, TYPE_MANAGER, locationService, partitionUpdateCodec, new HiveTypeTranslator(), new NodeVersion("test_version"), SqlStandardAccessControlMetadata::new);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), hivePartitionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HiveConfig()), new BoundedExecutor(executors, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getMaxInitialSplits(), config.getSplitLoaderConcurrency(), config.getMaxSplitsPerSecond(), config.getRecursiveDirWalkerEnabled(), null, config);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())), TYPE_MANAGER, config, locationService, partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config));
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
}
Also used : ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) HiveTestUtils.getDefaultHiveSelectiveFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories) FileSystem(org.apache.hadoop.fs.FileSystem) HdfsContext(io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext) ConnectorSplitManager(io.prestosql.spi.connector.ConnectorSplitManager) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) ConnectorPageSink(io.prestosql.spi.connector.ConnectorPageSink) Test(org.testng.annotations.Test) MaterializedResult.materializeSourceDataStream(io.prestosql.testing.MaterializedResult.materializeSourceDataStream) MaterializedResult(io.prestosql.testing.MaterializedResult) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TableNotFoundException(io.prestosql.spi.connector.TableNotFoundException) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) ConnectorPageSinkProvider(io.prestosql.spi.connector.ConnectorPageSinkProvider) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) Function(com.google.common.base.Function) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) SqlStandardAccessControlMetadata(io.prestosql.plugin.hive.security.SqlStandardAccessControlMetadata) ConnectorSplitSource(io.prestosql.spi.connector.ConnectorSplitSource) UUID(java.util.UUID) HiveTestUtils.getDefaultHiveFileWriterFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) UncheckedIOException(java.io.UncheckedIOException) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Table(io.prestosql.plugin.hive.metastore.Table) HiveTestUtils.getTypes(io.prestosql.plugin.hive.HiveTestUtils.getTypes) Optional(java.util.Optional) AbstractTestHive.getAllSplits(io.prestosql.plugin.hive.AbstractTestHive.getAllSplits) TestingNodeManager(io.prestosql.testing.TestingNodeManager) AbstractTestHive.filterNonHiddenColumnHandles(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnHandles) JsonCodec(io.airlift.json.JsonCodec) Database(io.prestosql.plugin.hive.metastore.Database) Slice(io.airlift.slice.Slice) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveTransaction(io.prestosql.plugin.hive.AbstractTestHive.HiveTransaction) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) AbstractTestHive.createTableProperties(io.prestosql.plugin.hive.AbstractTestHive.createTableProperties) Assert.assertEquals(org.testng.Assert.assertEquals) CounterStat(io.airlift.stats.CounterStat) AbstractTestHive.filterNonHiddenColumnMetadata(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) HiveMetastore(io.prestosql.plugin.hive.metastore.HiveMetastore) ExecutorService(java.util.concurrent.ExecutorService) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) AfterClass(org.testng.annotations.AfterClass) HiveTestUtils.getNoOpIndexCache(io.prestosql.plugin.hive.HiveTestUtils.getNoOpIndexCache) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) Executor(java.util.concurrent.Executor) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveTestUtils.getDefaultHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorIdentity(io.prestosql.spi.security.ConnectorIdentity) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) HiveTestUtils.getDefaultHiveDataStreamFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) PAGE_SORTER(io.prestosql.plugin.hive.HiveTestUtils.PAGE_SORTER) HostAndPort(com.google.common.net.HostAndPort) MaterializedRow(io.prestosql.testing.MaterializedRow) PrincipalPrivileges(io.prestosql.plugin.hive.metastore.PrincipalPrivileges) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Transaction(io.prestosql.plugin.hive.AbstractTestHive.Transaction) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) HiveTestUtils.getDefaultOrcFileWriterFactory(io.prestosql.plugin.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) ConnectorPageSourceProvider(io.prestosql.spi.connector.ConnectorPageSourceProvider) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) CounterStat(io.airlift.stats.CounterStat) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) TestingNodeManager(io.prestosql.testing.TestingNodeManager) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory)

Example 4 with TYPE_MANAGER

use of io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER in project hetu-core by openlookeng.

the class AbstractTestHiveFileFormats method createTestFilePresto.

public static FileSplit createTestFilePresto(String filePath, HiveStorageFormat storageFormat, HiveCompressionCodec compressionCodec, List<TestColumn> columns, ConnectorSession session, int numRows, HiveFileWriterFactory fileWriterFactory) {
    List<TestColumn> testColumns = columns;
    // filter out partition keys, which are not written to the file
    testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));
    List<Type> types = testColumns.stream().map(TestColumn::getType).map(HiveType::valueOf).map(type -> type.getType(TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
        pageBuilder.declarePosition();
        for (int columnNumber = 0; columnNumber < testColumns.size(); columnNumber++) {
            serializeObject(types.get(columnNumber), pageBuilder.getBlockBuilder(columnNumber), testColumns.get(columnNumber).getWriteValue(), testColumns.get(columnNumber).getObjectInspector(), false);
        }
    }
    Page page = pageBuilder.build();
    JobConf jobConf = new JobConf();
    configureCompression(jobConf, compressionCodec);
    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
    tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
    Optional<HiveFileWriter> fileWriter = fileWriterFactory.createFileWriter(new Path(filePath), testColumns.stream().map(TestColumn::getName).collect(toList()), StorageFormat.fromHiveStorageFormat(storageFormat), tableProperties, jobConf, session, Optional.empty(), Optional.empty());
    HiveFileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory"));
    hiveFileWriter.appendRows(page);
    hiveFileWriter.commit();
    return new FileSplit(new Path(filePath), 0, new File(filePath).length(), new String[0]);
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) StorageFormat(io.prestosql.plugin.hive.metastore.StorageFormat) JavaHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory.javaByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteObjectInspector) Text(org.apache.hadoop.io.Text) PrimitiveObjectInspectorFactory.javaLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector) Writable(org.apache.hadoop.io.Writable) Test(org.testng.annotations.Test) PrimitiveObjectInspectorFactory.javaTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector) Date(org.apache.hadoop.hive.common.type.Date) HiveTestUtils.isDistinctFrom(io.prestosql.plugin.hive.HiveTestUtils.isDistinctFrom) PrimitiveObjectInspectorFactory.javaDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDateObjectInspector) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) BigDecimal(java.math.BigDecimal) FileSplit(org.apache.hadoop.mapred.FileSplit) Predicates.not(com.google.common.base.Predicates.not) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) RowType(io.prestosql.spi.type.RowType) BigInteger(java.math.BigInteger) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) Assert.assertFalse(org.testng.Assert.assertFalse) Chars.isCharType(io.prestosql.spi.type.Chars.isCharType) PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector) PrimitiveObjectInspectorFactory.javaFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaFloatObjectInspector) PrimitiveObjectInspectorFactory.javaDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDoubleObjectInspector) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) TimestampType(io.prestosql.spi.type.TimestampType) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) StructuralTestUtil.mapBlockOf(io.prestosql.tests.StructuralTestUtil.mapBlockOf) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Iterables.filter(com.google.common.collect.Iterables.filter) StructuralTestUtil.decimalMapBlockOf(io.prestosql.tests.StructuralTestUtil.decimalMapBlockOf) PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector) Joiner(com.google.common.base.Joiner) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) JavaHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveCharObjectInspector) MethodHandle(java.lang.invoke.MethodHandle) Slice(io.airlift.slice.Slice) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) StructuralTestUtil.decimalArrayBlockOf(io.prestosql.tests.StructuralTestUtil.decimalArrayBlockOf) CharType(io.prestosql.spi.type.CharType) StructuralTestUtil.rowBlockOf(io.prestosql.tests.StructuralTestUtil.rowBlockOf) ArrayList(java.util.ArrayList) REGULAR(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR) PrimitiveObjectInspectorFactory.javaShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaShortObjectInspector) PARTITION_KEY(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) RecordCursor(io.prestosql.spi.connector.RecordCursor) StructuralTestUtil(io.prestosql.tests.StructuralTestUtil) SESSION(io.prestosql.plugin.hive.HiveTestUtils.SESSION) HiveTestUtils.mapType(io.prestosql.plugin.hive.HiveTestUtils.mapType) Arrays.fill(java.util.Arrays.fill) Properties(java.util.Properties) IOException(java.io.IOException) ObjectInspectorFactory.getStandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector) UTC(org.joda.time.DateTimeZone.UTC) MaterializedRow(io.prestosql.testing.MaterializedRow) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) File(java.io.File) SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) VARBINARY(io.prestosql.spi.type.VarbinaryType.VARBINARY) DateTimeTestingUtils.sqlTimestampOf(io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_DEFAULT_DYNAMIC_PARTITION(io.prestosql.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION) RecordWriter(org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter) Varchars.isVarcharType(io.prestosql.spi.type.Varchars.isVarcharType) Iterables.transform(com.google.common.collect.Iterables.transform) DecimalType(io.prestosql.spi.type.DecimalType) MaterializedResult.materializeSourceDataStream(io.prestosql.testing.MaterializedResult.materializeSourceDataStream) MaterializedResult(io.prestosql.testing.MaterializedResult) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Path(org.apache.hadoop.fs.Path) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) Type(io.prestosql.spi.type.Type) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) DateTimeFormat(org.joda.time.format.DateTimeFormat) ImmutableMap(com.google.common.collect.ImmutableMap) BlockBuilder(io.prestosql.spi.block.BlockBuilder) ArrayType(io.prestosql.spi.type.ArrayType) CharType.createCharType(io.prestosql.spi.type.CharType.createCharType) TINYINT(io.prestosql.spi.type.TinyintType.TINYINT) HiveOutputFormat(org.apache.hadoop.hive.ql.io.HiveOutputFormat) PageBuilder(io.prestosql.spi.PageBuilder) List(java.util.List) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) TypeInfoFactory.getCharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) Strings.padEnd(com.google.common.base.Strings.padEnd) HiveUtil.isStructuralType(io.prestosql.plugin.hive.HiveUtil.isStructuralType) PrimitiveObjectInspectorFactory.javaBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector) SqlDate(io.prestosql.spi.type.SqlDate) SerDeUtils.serializeObject(io.prestosql.plugin.hive.util.SerDeUtils.serializeObject) Assert.assertEquals(org.testng.Assert.assertEquals) Decimals(io.prestosql.spi.type.Decimals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) SqlDecimal(io.prestosql.spi.type.SqlDecimal) HdfsConfigurationInitializer.configureCompression(io.prestosql.plugin.hive.HdfsConfigurationInitializer.configureCompression) Float.intBitsToFloat(java.lang.Float.intBitsToFloat) SqlVarbinary(io.prestosql.spi.type.SqlVarbinary) Category(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) REAL(io.prestosql.spi.type.RealType.REAL) ObjectInspectorFactory.getStandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector) Block(io.prestosql.spi.block.Block) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) UTF_8(java.nio.charset.StandardCharsets.UTF_8) DateTime(org.joda.time.DateTime) Page(io.prestosql.spi.Page) ObjectInspectorFactory.getStandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector) JobConf(org.apache.hadoop.mapred.JobConf) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) SMALLINT(io.prestosql.spi.type.SmallintType.SMALLINT) DateType(io.prestosql.spi.type.DateType) Serializer(org.apache.hadoop.hive.serde2.Serializer) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) StructuralTestUtil.arrayBlockOf(io.prestosql.tests.StructuralTestUtil.arrayBlockOf) Assert.assertTrue(org.testng.Assert.assertTrue) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) Path(org.apache.hadoop.fs.Path) Page(io.prestosql.spi.Page) PageBuilder(io.prestosql.spi.PageBuilder) Properties(java.util.Properties) FileSplit(org.apache.hadoop.mapred.FileSplit) RowType(io.prestosql.spi.type.RowType) Chars.isCharType(io.prestosql.spi.type.Chars.isCharType) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) TimestampType(io.prestosql.spi.type.TimestampType) CharType(io.prestosql.spi.type.CharType) HiveTestUtils.mapType(io.prestosql.plugin.hive.HiveTestUtils.mapType) Varchars.isVarcharType(io.prestosql.spi.type.Varchars.isVarcharType) DecimalType(io.prestosql.spi.type.DecimalType) Type(io.prestosql.spi.type.Type) ArrayType(io.prestosql.spi.type.ArrayType) CharType.createCharType(io.prestosql.spi.type.CharType.createCharType) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) HiveUtil.isStructuralType(io.prestosql.plugin.hive.HiveUtil.isStructuralType) DateType(io.prestosql.spi.type.DateType) JobConf(org.apache.hadoop.mapred.JobConf) File(java.io.File)

Example 5 with TYPE_MANAGER

use of io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER in project hetu-core by openlookeng.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, Function<HiveConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        config.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(config, host, port);
    ExecutorService executors = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    ExecutorService executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, config, new NoHdfsAuthentication());
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executors, executorRefresh, config, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    metadataFactory = new HiveMetadataFactory(config, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), vacuumExecutorService, heartbeatService, vacuumExecutorService, TYPE_MANAGER, locationService, partitionUpdateCodec, new HiveTypeTranslator(), new NodeVersion("test_version"), SqlStandardAccessControlMetadata::new);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), hivePartitionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HiveConfig()), new BoundedExecutor(executors, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getMaxInitialSplits(), config.getSplitLoaderConcurrency(), config.getMaxSplitsPerSecond(), config.getRecursiveDirWalkerEnabled(), null, config);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())), TYPE_MANAGER, config, locationService, partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config));
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
}
Also used : ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) HiveTestUtils.getDefaultHiveSelectiveFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories) FileSystem(org.apache.hadoop.fs.FileSystem) HdfsContext(io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext) ConnectorSplitManager(io.prestosql.spi.connector.ConnectorSplitManager) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) ConnectorPageSink(io.prestosql.spi.connector.ConnectorPageSink) Test(org.testng.annotations.Test) MaterializedResult.materializeSourceDataStream(io.prestosql.testing.MaterializedResult.materializeSourceDataStream) MaterializedResult(io.prestosql.testing.MaterializedResult) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TableNotFoundException(io.prestosql.spi.connector.TableNotFoundException) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) ConnectorPageSinkProvider(io.prestosql.spi.connector.ConnectorPageSinkProvider) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) Function(com.google.common.base.Function) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) SqlStandardAccessControlMetadata(io.prestosql.plugin.hive.security.SqlStandardAccessControlMetadata) ConnectorSplitSource(io.prestosql.spi.connector.ConnectorSplitSource) UUID(java.util.UUID) HiveTestUtils.getDefaultHiveFileWriterFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) UncheckedIOException(java.io.UncheckedIOException) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Table(io.prestosql.plugin.hive.metastore.Table) HiveTestUtils.getTypes(io.prestosql.plugin.hive.HiveTestUtils.getTypes) Optional(java.util.Optional) AbstractTestHive.getAllSplits(io.prestosql.plugin.hive.AbstractTestHive.getAllSplits) TestingNodeManager(io.prestosql.testing.TestingNodeManager) AbstractTestHive.filterNonHiddenColumnHandles(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnHandles) JsonCodec(io.airlift.json.JsonCodec) Database(io.prestosql.plugin.hive.metastore.Database) Slice(io.airlift.slice.Slice) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveTransaction(io.prestosql.plugin.hive.AbstractTestHive.HiveTransaction) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) AbstractTestHive.createTableProperties(io.prestosql.plugin.hive.AbstractTestHive.createTableProperties) Assert.assertEquals(org.testng.Assert.assertEquals) CounterStat(io.airlift.stats.CounterStat) AbstractTestHive.filterNonHiddenColumnMetadata(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) HiveMetastore(io.prestosql.plugin.hive.metastore.HiveMetastore) ExecutorService(java.util.concurrent.ExecutorService) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) AfterClass(org.testng.annotations.AfterClass) HiveTestUtils.getNoOpIndexCache(io.prestosql.plugin.hive.HiveTestUtils.getNoOpIndexCache) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) Executor(java.util.concurrent.Executor) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveTestUtils.getDefaultHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorIdentity(io.prestosql.spi.security.ConnectorIdentity) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) HiveTestUtils.getDefaultHiveDataStreamFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) PAGE_SORTER(io.prestosql.plugin.hive.HiveTestUtils.PAGE_SORTER) HostAndPort(com.google.common.net.HostAndPort) MaterializedRow(io.prestosql.testing.MaterializedRow) PrincipalPrivileges(io.prestosql.plugin.hive.metastore.PrincipalPrivileges) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Transaction(io.prestosql.plugin.hive.AbstractTestHive.Transaction) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) HiveTestUtils.getDefaultOrcFileWriterFactory(io.prestosql.plugin.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) ConnectorPageSourceProvider(io.prestosql.spi.connector.ConnectorPageSourceProvider) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) CounterStat(io.airlift.stats.CounterStat) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) TestingNodeManager(io.prestosql.testing.TestingNodeManager) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)11 ImmutableMap (com.google.common.collect.ImmutableMap)11 TYPE_MANAGER (io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER)11 ConnectorPageSource (io.prestosql.spi.connector.ConnectorPageSource)11 ConnectorSession (io.prestosql.spi.connector.ConnectorSession)11 IOException (java.io.IOException)11 List (java.util.List)11 Optional (java.util.Optional)11 Assert.assertEquals (org.testng.Assert.assertEquals)11 Assert.assertTrue (org.testng.Assert.assertTrue)11 Test (org.testng.annotations.Test)11 Joiner (com.google.common.base.Joiner)9 Predicates.not (com.google.common.base.Predicates.not)9 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)9 Iterables.filter (com.google.common.collect.Iterables.filter)9 Iterables.transform (com.google.common.collect.Iterables.transform)9 Slices (io.airlift.slice.Slices)9 RecordCursor (io.prestosql.spi.connector.RecordCursor)9 File (java.io.File)9 Objects.requireNonNull (java.util.Objects.requireNonNull)9