Search in sources :

Example 1 with REGULAR

use of io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR in project boostkit-bigdata by kunpengcompute.

the class ParquetPageSourceFactory method createParquetPageSource.

public static ParquetPageSource createParquetPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, long fileSize, Properties schema, List<HiveColumnHandle> columns, boolean useParquetColumnNames, boolean failOnCorruptedParquetStatistics, DataSize maxReadBlockSize, TypeManager typeManager, TupleDomain<HiveColumnHandle> effectivePredicate, FileFormatDataSourceStats stats, DateTimeZone timeZone) {
    AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();
    ParquetDataSource dataSource = null;
    DateTimeZone readerTimeZone = timeZone;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(user, path, configuration);
        FSDataInputStream inputStream = hdfsEnvironment.doAs(user, () -> fileSystem.open(path));
        ParquetMetadata parquetMetadata = MetadataReader.readFooter(inputStream, path, fileSize);
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        dataSource = buildHdfsParquetDataSource(inputStream, path, fileSize, stats);
        String writerTimeZoneId = fileMetaData.getKeyValueMetaData().get(WRITER_TIME_ZONE_KEY);
        if (writerTimeZoneId != null && !writerTimeZoneId.equalsIgnoreCase(readerTimeZone.getID())) {
            readerTimeZone = DateTimeZone.forID(writerTimeZoneId);
        }
        List<org.apache.parquet.schema.Type> fields = columns.stream().filter(column -> column.getColumnType() == REGULAR).map(column -> getParquetType(column, fileSchema, useParquetColumnNames)).filter(Objects::nonNull).collect(toList());
        MessageType requestedSchema = new MessageType(fileSchema.getName(), fields);
        ImmutableList.Builder<BlockMetaData> footerBlocks = ImmutableList.builder();
        for (BlockMetaData block : parquetMetadata.getBlocks()) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            if (firstDataPage >= start && firstDataPage < start + length) {
                footerBlocks.add(block);
            }
        }
        Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
        TupleDomain<ColumnDescriptor> parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate);
        Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath);
        final ParquetDataSource finalDataSource = dataSource;
        ImmutableList.Builder<BlockMetaData> blocks = ImmutableList.builder();
        for (BlockMetaData block : footerBlocks.build()) {
            if (predicateMatches(parquetPredicate, block, finalDataSource, descriptorsByPath, parquetTupleDomain, failOnCorruptedParquetStatistics)) {
                blocks.add(block);
            }
        }
        MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema);
        ParquetReader parquetReader = new ParquetReader(Optional.ofNullable(fileMetaData.getCreatedBy()), messageColumnIO, blocks.build(), dataSource, readerTimeZone, systemMemoryContext, maxReadBlockSize);
        return new ParquetPageSource(parquetReader, fileSchema, messageColumnIO, typeManager, schema, columns, effectivePredicate, useParquetColumnNames);
    } catch (Exception e) {
        try {
            if (dataSource != null) {
                dataSource.close();
            }
        } catch (IOException ignored) {
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        if (e instanceof ParquetCorruptionException) {
            throw new PrestoException(HIVE_BAD_DATA, e);
        }
        if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) {
            throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e);
        }
        String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e instanceof BlockMissingException) {
            throw new PrestoException(HIVE_MISSING_DATA, message, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
    }
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) ParquetTypeUtils.getColumnIO(io.prestosql.parquet.ParquetTypeUtils.getColumnIO) HiveSessionProperties.isUseParquetColumnNames(io.prestosql.plugin.hive.HiveSessionProperties.isUseParquetColumnNames) FileSystem(org.apache.hadoop.fs.FileSystem) HivePartitionKey(io.prestosql.plugin.hive.HivePartitionKey) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) DataReader(com.huawei.boostkit.omnidata.reader.DataReader) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) ParquetTypeUtils.getParquetTypeByName(io.prestosql.parquet.ParquetTypeUtils.getParquetTypeByName) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PredicateUtils.buildPredicate(io.prestosql.parquet.predicate.PredicateUtils.buildPredicate) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) HiveConfig(io.prestosql.plugin.hive.HiveConfig) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) Path(org.apache.hadoop.fs.Path) OMNIDATA_CLIENT_TARGET_LIST(com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TARGET_LIST) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) DataReaderFactory(com.huawei.boostkit.omnidata.reader.DataReaderFactory) HiveSessionProperties.isFailOnCorruptedParquetStatistics(io.prestosql.plugin.hive.HiveSessionProperties.isFailOnCorruptedParquetStatistics) PageSourceUtil.buildPushdownContext(io.prestosql.plugin.hive.util.PageSourceUtil.buildPushdownContext) PrestoException(io.prestosql.spi.PrestoException) HdfsParquetDataSource.buildHdfsParquetDataSource(io.prestosql.plugin.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) HivePushDownPageSource(io.prestosql.plugin.hive.HivePushDownPageSource) MetadataReader(io.prestosql.parquet.reader.MetadataReader) FileFormatDataSourceStats(io.prestosql.plugin.hive.FileFormatDataSourceStats) FileNotFoundException(java.io.FileNotFoundException) String.format(java.lang.String.format) DataSource(com.huawei.boostkit.omnidata.model.datasource.DataSource) Objects(java.util.Objects) MessageType(org.apache.parquet.schema.MessageType) DataSize(io.airlift.units.DataSize) List(java.util.List) HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) HiveUtil.shouldUseRecordReaderFromInputFormat(io.prestosql.plugin.hive.HiveUtil.shouldUseRecordReaderFromInputFormat) HIVE_BAD_DATA(io.prestosql.plugin.hive.HiveErrorCode.HIVE_BAD_DATA) ColumnDescriptor(org.apache.parquet.column.ColumnDescriptor) BlockMetaData(org.apache.parquet.hadoop.metadata.BlockMetaData) PredicateUtils.predicateMatches(io.prestosql.parquet.predicate.PredicateUtils.predicateMatches) Domain(io.prestosql.spi.predicate.Domain) Entry(java.util.Map.Entry) Optional(java.util.Optional) IndexMetadata(io.prestosql.spi.heuristicindex.IndexMetadata) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) SplitMetadata(io.prestosql.spi.heuristicindex.SplitMetadata) MessageColumnIO(org.apache.parquet.io.MessageColumnIO) HIVE_CANNOT_OPEN_SPLIT(io.prestosql.plugin.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) ParquetReader(io.prestosql.parquet.reader.ParquetReader) HiveSessionProperties(io.prestosql.plugin.hive.HiveSessionProperties) ParquetTypeUtils.getDescriptors(io.prestosql.parquet.ParquetTypeUtils.getDescriptors) OptionalInt(java.util.OptionalInt) TaskSource(com.huawei.boostkit.omnidata.model.TaskSource) Predicate(io.prestosql.parquet.predicate.Predicate) Inject(javax.inject.Inject) HiveUtil.getDeserializerClassName(io.prestosql.plugin.hive.HiveUtil.getDeserializerClassName) HdfsEnvironment(io.prestosql.plugin.hive.HdfsEnvironment) HIVE_MISSING_DATA(io.prestosql.plugin.hive.HiveErrorCode.HIVE_MISSING_DATA) REGULAR(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) DynamicFilterSupplier(io.prestosql.spi.dynamicfilter.DynamicFilterSupplier) HivePageSourceFactory(io.prestosql.plugin.hive.HivePageSourceFactory) Properties(java.util.Properties) DeleteDeltaLocations(io.prestosql.plugin.hive.DeleteDeltaLocations) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TypeManager(io.prestosql.spi.type.TypeManager) Page(io.prestosql.spi.Page) IOException(java.io.IOException) PRIMITIVE(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE) ParquetDataSource(io.prestosql.parquet.ParquetDataSource) Collectors.toList(java.util.stream.Collectors.toList) HiveSessionProperties.getParquetMaxReadBlockSize(io.prestosql.plugin.hive.HiveSessionProperties.getParquetMaxReadBlockSize) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) FileMetaData(org.apache.parquet.hadoop.metadata.FileMetaData) ParquetMetadata(org.apache.parquet.hadoop.metadata.ParquetMetadata) OpenLooKengDeserializer(com.huawei.boostkit.omnidata.decode.impl.OpenLooKengDeserializer) BlockMetaData(org.apache.parquet.hadoop.metadata.BlockMetaData) ParquetMetadata(org.apache.parquet.hadoop.metadata.ParquetMetadata) ImmutableList(com.google.common.collect.ImmutableList) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) FileNotFoundException(java.io.FileNotFoundException) PrestoException(io.prestosql.spi.PrestoException) MessageColumnIO(org.apache.parquet.io.MessageColumnIO) PredicateUtils.buildPredicate(io.prestosql.parquet.predicate.PredicateUtils.buildPredicate) Predicate(io.prestosql.parquet.predicate.Predicate) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) FileSystem(org.apache.hadoop.fs.FileSystem) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) FileMetaData(org.apache.parquet.hadoop.metadata.FileMetaData) MessageType(org.apache.parquet.schema.MessageType) HdfsParquetDataSource.buildHdfsParquetDataSource(io.prestosql.plugin.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource) ParquetDataSource(io.prestosql.parquet.ParquetDataSource) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) ColumnDescriptor(org.apache.parquet.column.ColumnDescriptor) ParquetReader(io.prestosql.parquet.reader.ParquetReader) IOException(java.io.IOException) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) DateTimeZone(org.joda.time.DateTimeZone) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) PrestoException(io.prestosql.spi.PrestoException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) MessageType(org.apache.parquet.schema.MessageType) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream)

Example 2 with REGULAR

use of io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR in project hetu-core by openlookeng.

the class ParquetPageSourceFactory method createParquetPageSource.

public static ParquetPageSource createParquetPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, long fileSize, Properties schema, List<HiveColumnHandle> columns, boolean useParquetColumnNames, boolean failOnCorruptedParquetStatistics, DataSize maxReadBlockSize, TypeManager typeManager, TupleDomain<HiveColumnHandle> effectivePredicate, FileFormatDataSourceStats stats, DateTimeZone timeZone) {
    AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();
    ParquetDataSource dataSource = null;
    DateTimeZone readerTimeZone = timeZone;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(user, path, configuration);
        FSDataInputStream inputStream = hdfsEnvironment.doAs(user, () -> fileSystem.open(path));
        ParquetMetadata parquetMetadata = MetadataReader.readFooter(inputStream, path, fileSize);
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        dataSource = buildHdfsParquetDataSource(inputStream, path, fileSize, stats);
        String writerTimeZoneId = fileMetaData.getKeyValueMetaData().get(WRITER_TIME_ZONE_KEY);
        if (writerTimeZoneId != null && !writerTimeZoneId.equalsIgnoreCase(readerTimeZone.getID())) {
            readerTimeZone = DateTimeZone.forID(writerTimeZoneId);
        }
        List<org.apache.parquet.schema.Type> fields = columns.stream().filter(column -> column.getColumnType() == REGULAR).map(column -> getParquetType(column, fileSchema, useParquetColumnNames)).filter(Objects::nonNull).collect(toList());
        MessageType requestedSchema = new MessageType(fileSchema.getName(), fields);
        ImmutableList.Builder<BlockMetaData> footerBlocks = ImmutableList.builder();
        for (BlockMetaData block : parquetMetadata.getBlocks()) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            if (firstDataPage >= start && firstDataPage < start + length) {
                footerBlocks.add(block);
            }
        }
        Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
        TupleDomain<ColumnDescriptor> parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate);
        Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath);
        final ParquetDataSource finalDataSource = dataSource;
        ImmutableList.Builder<BlockMetaData> blocks = ImmutableList.builder();
        for (BlockMetaData block : footerBlocks.build()) {
            if (predicateMatches(parquetPredicate, block, finalDataSource, descriptorsByPath, parquetTupleDomain, failOnCorruptedParquetStatistics)) {
                blocks.add(block);
            }
        }
        MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema);
        ParquetReader parquetReader = new ParquetReader(Optional.ofNullable(fileMetaData.getCreatedBy()), messageColumnIO, blocks.build(), dataSource, readerTimeZone, systemMemoryContext, maxReadBlockSize);
        return new ParquetPageSource(parquetReader, fileSchema, messageColumnIO, typeManager, schema, columns, effectivePredicate, useParquetColumnNames);
    } catch (Exception e) {
        try {
            if (dataSource != null) {
                dataSource.close();
            }
        } catch (IOException ignored) {
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        if (e instanceof ParquetCorruptionException) {
            throw new PrestoException(HIVE_BAD_DATA, e);
        }
        if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) {
            throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e);
        }
        String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e instanceof BlockMissingException) {
            throw new PrestoException(HIVE_MISSING_DATA, message, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
    }
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) ParquetTypeUtils.getColumnIO(io.prestosql.parquet.ParquetTypeUtils.getColumnIO) HiveSessionProperties.isUseParquetColumnNames(io.prestosql.plugin.hive.HiveSessionProperties.isUseParquetColumnNames) FileSystem(org.apache.hadoop.fs.FileSystem) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) ParquetTypeUtils.getParquetTypeByName(io.prestosql.parquet.ParquetTypeUtils.getParquetTypeByName) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PredicateUtils.buildPredicate(io.prestosql.parquet.predicate.PredicateUtils.buildPredicate) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) HiveConfig(io.prestosql.plugin.hive.HiveConfig) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) Path(org.apache.hadoop.fs.Path) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) HiveSessionProperties.isFailOnCorruptedParquetStatistics(io.prestosql.plugin.hive.HiveSessionProperties.isFailOnCorruptedParquetStatistics) PrestoException(io.prestosql.spi.PrestoException) HdfsParquetDataSource.buildHdfsParquetDataSource(io.prestosql.plugin.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) MetadataReader(io.prestosql.parquet.reader.MetadataReader) FileFormatDataSourceStats(io.prestosql.plugin.hive.FileFormatDataSourceStats) FileNotFoundException(java.io.FileNotFoundException) String.format(java.lang.String.format) Objects(java.util.Objects) MessageType(org.apache.parquet.schema.MessageType) DataSize(io.airlift.units.DataSize) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) HiveUtil.shouldUseRecordReaderFromInputFormat(io.prestosql.plugin.hive.HiveUtil.shouldUseRecordReaderFromInputFormat) HIVE_BAD_DATA(io.prestosql.plugin.hive.HiveErrorCode.HIVE_BAD_DATA) ColumnDescriptor(org.apache.parquet.column.ColumnDescriptor) BlockMetaData(org.apache.parquet.hadoop.metadata.BlockMetaData) PredicateUtils.predicateMatches(io.prestosql.parquet.predicate.PredicateUtils.predicateMatches) Domain(io.prestosql.spi.predicate.Domain) Entry(java.util.Map.Entry) Optional(java.util.Optional) IndexMetadata(io.prestosql.spi.heuristicindex.IndexMetadata) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) SplitMetadata(io.prestosql.spi.heuristicindex.SplitMetadata) MessageColumnIO(org.apache.parquet.io.MessageColumnIO) HIVE_CANNOT_OPEN_SPLIT(io.prestosql.plugin.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) ParquetReader(io.prestosql.parquet.reader.ParquetReader) ParquetTypeUtils.getDescriptors(io.prestosql.parquet.ParquetTypeUtils.getDescriptors) Predicate(io.prestosql.parquet.predicate.Predicate) Inject(javax.inject.Inject) HiveUtil.getDeserializerClassName(io.prestosql.plugin.hive.HiveUtil.getDeserializerClassName) HdfsEnvironment(io.prestosql.plugin.hive.HdfsEnvironment) HIVE_MISSING_DATA(io.prestosql.plugin.hive.HiveErrorCode.HIVE_MISSING_DATA) REGULAR(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) DynamicFilterSupplier(io.prestosql.spi.dynamicfilter.DynamicFilterSupplier) HivePageSourceFactory(io.prestosql.plugin.hive.HivePageSourceFactory) Properties(java.util.Properties) DeleteDeltaLocations(io.prestosql.plugin.hive.DeleteDeltaLocations) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TypeManager(io.prestosql.spi.type.TypeManager) IOException(java.io.IOException) PRIMITIVE(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE) ParquetDataSource(io.prestosql.parquet.ParquetDataSource) Collectors.toList(java.util.stream.Collectors.toList) HiveSessionProperties.getParquetMaxReadBlockSize(io.prestosql.plugin.hive.HiveSessionProperties.getParquetMaxReadBlockSize) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) FileMetaData(org.apache.parquet.hadoop.metadata.FileMetaData) ParquetMetadata(org.apache.parquet.hadoop.metadata.ParquetMetadata) BlockMetaData(org.apache.parquet.hadoop.metadata.BlockMetaData) ParquetMetadata(org.apache.parquet.hadoop.metadata.ParquetMetadata) ImmutableList(com.google.common.collect.ImmutableList) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) FileNotFoundException(java.io.FileNotFoundException) PrestoException(io.prestosql.spi.PrestoException) MessageColumnIO(org.apache.parquet.io.MessageColumnIO) PredicateUtils.buildPredicate(io.prestosql.parquet.predicate.PredicateUtils.buildPredicate) Predicate(io.prestosql.parquet.predicate.Predicate) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) FileSystem(org.apache.hadoop.fs.FileSystem) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) FileMetaData(org.apache.parquet.hadoop.metadata.FileMetaData) MessageType(org.apache.parquet.schema.MessageType) HdfsParquetDataSource.buildHdfsParquetDataSource(io.prestosql.plugin.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource) ParquetDataSource(io.prestosql.parquet.ParquetDataSource) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) ColumnDescriptor(org.apache.parquet.column.ColumnDescriptor) ParquetReader(io.prestosql.parquet.reader.ParquetReader) IOException(java.io.IOException) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) DateTimeZone(org.joda.time.DateTimeZone) BlockMissingException(org.apache.hadoop.hdfs.BlockMissingException) PrestoException(io.prestosql.spi.PrestoException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) ParquetCorruptionException(io.prestosql.parquet.ParquetCorruptionException) MessageType(org.apache.parquet.schema.MessageType) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream)

Aggregations

Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 Strings.nullToEmpty (com.google.common.base.Strings.nullToEmpty)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 DataSize (io.airlift.units.DataSize)2 AggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext)2 AggregatedMemoryContext.newSimpleAggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext)2 ParquetCorruptionException (io.prestosql.parquet.ParquetCorruptionException)2 ParquetDataSource (io.prestosql.parquet.ParquetDataSource)2 ParquetTypeUtils.getColumnIO (io.prestosql.parquet.ParquetTypeUtils.getColumnIO)2 ParquetTypeUtils.getDescriptors (io.prestosql.parquet.ParquetTypeUtils.getDescriptors)2 ParquetTypeUtils.getParquetTypeByName (io.prestosql.parquet.ParquetTypeUtils.getParquetTypeByName)2 RichColumnDescriptor (io.prestosql.parquet.RichColumnDescriptor)2 Predicate (io.prestosql.parquet.predicate.Predicate)2 PredicateUtils.buildPredicate (io.prestosql.parquet.predicate.PredicateUtils.buildPredicate)2 PredicateUtils.predicateMatches (io.prestosql.parquet.predicate.PredicateUtils.predicateMatches)2 MetadataReader (io.prestosql.parquet.reader.MetadataReader)2 ParquetReader (io.prestosql.parquet.reader.ParquetReader)2 DeleteDeltaLocations (io.prestosql.plugin.hive.DeleteDeltaLocations)2