Examples with ParquetMetadata org.apache.parquet.hadoop.metadata.ParquetMetadata used on opensource projects

Example 71 with ParquetMetadata

use of org.apache.parquet.hadoop.metadata.ParquetMetadata in project drill by apache.

the class AbstractParquetScanBatchCreator method getBatch.

protected ScanBatch getBatch(ExecutorFragmentContext context, AbstractParquetRowGroupScan rowGroupScan, OperatorContext oContext) throws ExecutionSetupException {
    final ColumnExplorer columnExplorer = new ColumnExplorer(context.getOptions(), rowGroupScan.getColumns());
    if (!columnExplorer.isStarQuery()) {
        rowGroupScan = rowGroupScan.copy(columnExplorer.getTableColumns());
        rowGroupScan.setOperatorId(rowGroupScan.getOperatorId());
    }
    AbstractDrillFileSystemManager fsManager = getDrillFileSystemCreator(oContext, context.getOptions());
    // keep footers in a map to avoid re-reading them
    Map<Path, ParquetMetadata> footers = new HashMap<>();
    List<CommonParquetRecordReader> readers = new LinkedList<>();
    List<Map<String, String>> implicitColumns = new ArrayList<>();
    Map<String, String> mapWithMaxColumns = new LinkedHashMap<>();
    ParquetReaderConfig readerConfig = rowGroupScan.getReaderConfig();
    // to be scanned in case ALL row groups are pruned out
    RowGroupReadEntry firstRowGroup = null;
    ParquetMetadata firstFooter = null;
    // for stats
    long rowGroupsPruned = 0;
    try {
        LogicalExpression filterExpr = rowGroupScan.getFilter();
        boolean doRuntimePruning = // was a filter given ?   And it is not just a "TRUE" predicate
        filterExpr != null && !((filterExpr instanceof ValueExpressions.BooleanExpression) && ((ValueExpressions.BooleanExpression) filterExpr).getBoolean());
        // Runtime pruning: Avoid recomputing metadata objects for each row-group in case they use the same file
        // by keeping the following objects computed earlier (relies on same file being in consecutive rowgroups)
        Path prevRowGroupPath = null;
        Metadata_V4.ParquetTableMetadata_v4 tableMetadataV4 = null;
        Metadata_V4.ParquetFileAndRowCountMetadata fileMetadataV4 = null;
        FilterPredicate<?> filterPredicate = null;
        Set<SchemaPath> schemaPathsInExpr = null;
        Set<SchemaPath> columnsInExpr = null;
        // for debug/info logging
        long totalPruneTime = 0;
        long totalRowGroups = rowGroupScan.getRowGroupReadEntries().size();
        Stopwatch pruneTimer = Stopwatch.createUnstarted();
        // If pruning - Prepare the predicate and the columns before the FOR LOOP
        if (doRuntimePruning) {
            filterPredicate = AbstractGroupScanWithMetadata.getFilterPredicate(filterExpr, context, context.getFunctionRegistry(), context.getOptions(), true, true, /* supports file implicit columns */
            rowGroupScan.getSchema());
            // Extract only the relevant columns from the filter (sans implicit columns, if any)
            schemaPathsInExpr = filterExpr.accept(FilterEvaluatorUtils.FieldReferenceFinder.INSTANCE, null);
            columnsInExpr = new HashSet<>();
            String partitionColumnLabel = context.getOptions().getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val;
            for (SchemaPath path : schemaPathsInExpr) {
                if (rowGroupScan.supportsFileImplicitColumns() && path.toString().matches(partitionColumnLabel + "\\d+")) {
                    // skip implicit columns like dir0, dir1
                    continue;
                }
                columnsInExpr.add(SchemaPath.getSimplePath(path.getRootSegmentPath()));
            }
            // just in case: if no columns - cancel pruning
            doRuntimePruning = !columnsInExpr.isEmpty();
        }
        for (RowGroupReadEntry rowGroup : rowGroupScan.getRowGroupReadEntries()) {
            /*
        Here we could store a map from file names to footers, to prevent re-reading the footer for each row group in a file
        TODO - to prevent reading the footer again in the parquet record reader (it is read earlier in the ParquetStorageEngine)
        we should add more information to the RowGroupInfo that will be populated upon the first read to
        provide the reader with all of the file meta-data it needs
        These fields will be added to the constructor below
        */
            Stopwatch timer = logger.isTraceEnabled() ? Stopwatch.createUnstarted() : null;
            DrillFileSystem fs = fsManager.get(rowGroupScan.getFsConf(rowGroup), rowGroup.getPath());
            if (!footers.containsKey(rowGroup.getPath())) {
                if (timer != null) {
                    timer.start();
                }
                ParquetMetadata footer = readFooter(fs.getConf(), rowGroup.getPath(), readerConfig);
                if (timer != null) {
                    long timeToRead = timer.elapsed(TimeUnit.MICROSECONDS);
                    logger.trace("ParquetTrace,Read Footer,{},{},{},{},{},{},{}", "", rowGroup.getPath(), "", 0, 0, 0, timeToRead);
                }
                footers.put(rowGroup.getPath(), footer);
            }
            ParquetMetadata footer = footers.get(rowGroup.getPath());
            // 
            if (doRuntimePruning) {
                // skip when no filter or filter is TRUE
                pruneTimer.start();
                // 
                // Perform the Run-Time Pruning - i.e. Skip/prune this row group if the match fails
                // 
                // default (in case of exception) - do not prune this row group
                RowsMatch matchResult = RowsMatch.ALL;
                if (rowGroup.isEmpty()) {
                    matchResult = RowsMatch.NONE;
                } else {
                    int rowGroupIndex = rowGroup.getRowGroupIndex();
                    long footerRowCount = footer.getBlocks().get(rowGroupIndex).getRowCount();
                    // When starting a new file, or at the first time - Initialize the path specific metadata
                    if (!rowGroup.getPath().equals(prevRowGroupPath)) {
                        // Create a table metadata (V4)
                        tableMetadataV4 = new Metadata_V4.ParquetTableMetadata_v4();
                        // The file status for this file
                        FileStatus fileStatus = fs.getFileStatus(rowGroup.getPath());
                        // The file metadata (only for the columns used in the filter)
                        fileMetadataV4 = Metadata.getParquetFileMetadata_v4(tableMetadataV4, footer, fileStatus, fs, false, true, columnsInExpr, readerConfig);
                        // for next time
                        prevRowGroupPath = rowGroup.getPath();
                    }
                    MetadataBase.RowGroupMetadata rowGroupMetadata = fileMetadataV4.getFileMetadata().getRowGroups().get(rowGroup.getRowGroupIndex());
                    Map<SchemaPath, ColumnStatistics<?>> columnsStatistics = ParquetTableMetadataUtils.getRowGroupColumnStatistics(tableMetadataV4, rowGroupMetadata);
                    try {
                        Map<SchemaPath, TypeProtos.MajorType> intermediateColumns = ParquetTableMetadataUtils.getIntermediateFields(tableMetadataV4, rowGroupMetadata);
                        Map<SchemaPath, TypeProtos.MajorType> rowGroupFields = ParquetTableMetadataUtils.getRowGroupFields(tableMetadataV4, rowGroupMetadata);
                        TupleMetadata rowGroupSchema = new TupleSchema();
                        rowGroupFields.forEach((schemaPath, majorType) -> SchemaPathUtils.addColumnMetadata(rowGroupSchema, schemaPath, majorType, intermediateColumns));
                        // updates filter predicate to add required casts for the case when row group schema differs from the table schema
                        if (!rowGroupSchema.isEquivalent(rowGroupScan.getSchema())) {
                            filterPredicate = AbstractGroupScanWithMetadata.getFilterPredicate(filterExpr, context, context.getFunctionRegistry(), context.getOptions(), true, true, /* supports file implicit columns */
                            rowGroupSchema);
                        }
                        matchResult = FilterEvaluatorUtils.matches(filterPredicate, columnsStatistics, footerRowCount, rowGroupSchema, schemaPathsInExpr);
                        // collect logging info
                        long timeToRead = pruneTimer.elapsed(TimeUnit.MICROSECONDS);
                        totalPruneTime += timeToRead;
                        // trace each single row group
                        logger.trace(// trace each single row group
                        "Run-time pruning: {} row-group {} (RG index: {} row count: {}), took {} usec", matchResult == RowsMatch.NONE ? "Excluded" : "Included", rowGroup.getPath(), rowGroupIndex, footerRowCount, timeToRead);
                    } catch (Exception e) {
                        // in case some unexpected exception is raised
                        logger.warn("Run-time pruning check failed - {}. Skip pruning rowgroup - {}", e.getMessage(), rowGroup.getPath());
                        logger.debug("Failure during run-time pruning: {}", e.getMessage(), e);
                    }
                }
                pruneTimer.stop();
                pruneTimer.reset();
                // If this row group failed the match - skip it (i.e., no reader for this rowgroup)
                if (matchResult == RowsMatch.NONE) {
                    // one more RG was pruned
                    rowGroupsPruned++;
                    if (firstRowGroup == null) {
                        // keep the first RG, to be used in case all row groups are pruned
                        firstRowGroup = rowGroup;
                        firstFooter = footer;
                    }
                    // This Row group does not comply with the filter - prune it out and check the next Row Group
                    continue;
                }
            }
            mapWithMaxColumns = createReaderAndImplicitColumns(context, rowGroupScan, oContext, columnExplorer, readers, implicitColumns, mapWithMaxColumns, rowGroup, fs, footer, false);
        }
        // in case all row groups were pruned out - create a single reader for the first one (so that the schema could be returned)
        if (readers.isEmpty() && firstRowGroup != null) {
            DrillFileSystem fs = fsManager.get(rowGroupScan.getFsConf(firstRowGroup), firstRowGroup.getPath());
            mapWithMaxColumns = createReaderAndImplicitColumns(context, rowGroupScan, oContext, columnExplorer, readers, implicitColumns, mapWithMaxColumns, firstRowGroup, fs, firstFooter, true);
        }
        // do some logging, if relevant
        if (totalPruneTime > 0) {
            logger.info("Finished parquet_runtime_pruning in {} usec. Out of given {} rowgroups, {} were pruned. {}", totalPruneTime, totalRowGroups, rowGroupsPruned, totalRowGroups == rowGroupsPruned ? "ALL_PRUNED !!" : "");
        }
        // Update stats (same in every reader - the others would just overwrite the stats)
        for (CommonParquetRecordReader rr : readers) {
            rr.updateRowGroupsStats(totalRowGroups, rowGroupsPruned);
        }
    } catch (IOException | InterruptedException e) {
        throw new ExecutionSetupException(e);
    }
    // all readers should have the same number of implicit columns, add missing ones with value null
    Map<String, String> diff = Maps.transformValues(mapWithMaxColumns, Functions.constant(null));
    for (Map<String, String> map : implicitColumns) {
        map.putAll(Maps.difference(map, diff).entriesOnlyOnRight());
    }
    return new ScanBatch(context, oContext, readers, implicitColumns);
}

Example 72 with ParquetMetadata

use of org.apache.parquet.hadoop.metadata.ParquetMetadata in project drill by apache.

the class FooterGatherer method readFooter.

/**
 * An updated footer reader that tries to read the entire footer without knowing the length.
 * This should reduce the amount of seek/read roundtrips in most workloads.
 * @param config configuration for file system
 * @param status file status
 * @return Footer
 * @throws IOException
 */
public static Footer readFooter(final Configuration config, final FileStatus status) throws IOException {
    final FileSystem fs = status.getPath().getFileSystem(config);
    try (FSDataInputStream file = fs.open(status.getPath())) {
        final long fileLength = status.getLen();
        Preconditions.checkArgument(fileLength >= MIN_FILE_SIZE, "%s is not a Parquet file (too small)", status.getPath());
        int len = (int) Math.min(fileLength, (long) DEFAULT_READ_SIZE);
        byte[] footerBytes = new byte[len];
        readFully(file, fileLength - len, footerBytes, 0, len);
        checkMagicBytes(status, footerBytes, footerBytes.length - ParquetFileWriter.MAGIC.length);
        final int size = BytesUtils.readIntLittleEndian(footerBytes, footerBytes.length - FOOTER_METADATA_SIZE);
        if (size > footerBytes.length - FOOTER_METADATA_SIZE) {
            // if the footer is larger than our initial read, we need to read the rest.
            byte[] origFooterBytes = footerBytes;
            int origFooterRead = origFooterBytes.length - FOOTER_METADATA_SIZE;
            footerBytes = new byte[size];
            readFully(file, fileLength - size - FOOTER_METADATA_SIZE, footerBytes, 0, size - origFooterRead);
            System.arraycopy(origFooterBytes, 0, footerBytes, size - origFooterRead, origFooterRead);
        } else {
            int start = footerBytes.length - (size + FOOTER_METADATA_SIZE);
            footerBytes = ArrayUtils.subarray(footerBytes, start, start + size);
        }
        final ByteArrayInputStream from = new ByteArrayInputStream(footerBytes);
        ParquetMetadata metadata = ParquetFormatPlugin.parquetMetadataConverter.readParquetMetadata(from, NO_FILTER);
        Footer footer = new Footer(status.getPath(), metadata);
        return footer;
    }
}

Example 73 with ParquetMetadata

use of org.apache.parquet.hadoop.metadata.ParquetMetadata in project presto by prestodb.

the class ParquetPageSourceFactory method createParquetPageSource.

public static ConnectorPageSource createParquetPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, long fileSize, List<HiveColumnHandle> columns, SchemaTableName tableName, boolean useParquetColumnNames, DataSize maxReadBlockSize, boolean batchReaderEnabled, boolean verificationEnabled, TypeManager typeManager, StandardFunctionResolution functionResolution, TupleDomain<HiveColumnHandle> effectivePredicate, FileFormatDataSourceStats stats, HiveFileContext hiveFileContext, ParquetMetadataSource parquetMetadataSource, boolean columnIndexFilterEnabled) {
    AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();
    ParquetDataSource dataSource = null;
    try {
        FSDataInputStream inputStream = hdfsEnvironment.getFileSystem(user, path, configuration).openFile(path, hiveFileContext);
        dataSource = buildHdfsParquetDataSource(inputStream, path, stats);
        ParquetMetadata parquetMetadata = parquetMetadataSource.getParquetMetadata(dataSource, fileSize, hiveFileContext.isCacheable()).getParquetMetadata();
        if (!columns.isEmpty() && columns.stream().allMatch(hiveColumnHandle -> hiveColumnHandle.getColumnType() == AGGREGATED)) {
            return new AggregatedParquetPageSource(columns, parquetMetadata, typeManager, functionResolution);
        }
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        Optional<MessageType> message = columns.stream().filter(column -> column.getColumnType() == REGULAR || isPushedDownSubfield(column)).map(column -> getColumnType(typeManager.getType(column.getTypeSignature()), fileSchema, useParquetColumnNames, column, tableName, path)).filter(Optional::isPresent).map(Optional::get).map(type -> new MessageType(fileSchema.getName(), type)).reduce(MessageType::union);
        MessageType requestedSchema = message.orElse(new MessageType(fileSchema.getName(), ImmutableList.of()));
        ImmutableList.Builder<BlockMetaData> footerBlocks = ImmutableList.builder();
        for (BlockMetaData block : parquetMetadata.getBlocks()) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            if (firstDataPage >= start && firstDataPage < start + length) {
                footerBlocks.add(block);
            }
        }
        Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
        TupleDomain<ColumnDescriptor> parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate);
        Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath);
        final ParquetDataSource finalDataSource = dataSource;
        ImmutableList.Builder<BlockMetaData> blocks = ImmutableList.builder();
        List<ColumnIndexStore> blockIndexStores = new ArrayList<>();
        for (BlockMetaData block : footerBlocks.build()) {
            Optional<ColumnIndexStore> columnIndexStore = ColumnIndexFilterUtils.getColumnIndexStore(parquetPredicate, finalDataSource, block, descriptorsByPath, columnIndexFilterEnabled);
            if (predicateMatches(parquetPredicate, block, finalDataSource, descriptorsByPath, parquetTupleDomain, columnIndexStore, columnIndexFilterEnabled)) {
                blocks.add(block);
                blockIndexStores.add(columnIndexStore.orElse(null));
                hiveFileContext.incrementCounter("parquet.blocksRead", 1);
                hiveFileContext.incrementCounter("parquet.rowsRead", block.getRowCount());
                hiveFileContext.incrementCounter("parquet.totalBytesRead", block.getTotalByteSize());
            } else {
                hiveFileContext.incrementCounter("parquet.blocksSkipped", 1);
                hiveFileContext.incrementCounter("parquet.rowsSkipped", block.getRowCount());
                hiveFileContext.incrementCounter("parquet.totalBytesSkipped", block.getTotalByteSize());
            }
        }
        MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema);
        ParquetReader parquetReader = new ParquetReader(messageColumnIO, blocks.build(), dataSource, systemMemoryContext, maxReadBlockSize, batchReaderEnabled, verificationEnabled, parquetPredicate, blockIndexStores, columnIndexFilterEnabled);
        ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
        ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
        ImmutableList.Builder<Optional<Field>> fieldsBuilder = ImmutableList.builder();
        for (HiveColumnHandle column : columns) {
            checkArgument(column.getColumnType() == REGULAR || column.getColumnType() == SYNTHESIZED, "column type must be regular or synthesized column");
            String name = column.getName();
            Type type = typeManager.getType(column.getTypeSignature());
            namesBuilder.add(name);
            typesBuilder.add(type);
            if (column.getColumnType() == SYNTHESIZED) {
                Subfield pushedDownSubfield = getPushedDownSubfield(column);
                List<String> nestedColumnPath = nestedColumnPath(pushedDownSubfield);
                Optional<ColumnIO> columnIO = findNestedColumnIO(lookupColumnByName(messageColumnIO, pushedDownSubfield.getRootName()), nestedColumnPath);
                if (columnIO.isPresent()) {
                    fieldsBuilder.add(constructField(type, columnIO.get()));
                } else {
                    fieldsBuilder.add(Optional.empty());
                }
            } else if (getParquetType(type, fileSchema, useParquetColumnNames, column, tableName, path).isPresent()) {
                String columnName = useParquetColumnNames ? name : fileSchema.getFields().get(column.getHiveColumnIndex()).getName();
                fieldsBuilder.add(constructField(type, lookupColumnByName(messageColumnIO, columnName)));
            } else {
                fieldsBuilder.add(Optional.empty());
            }
        }
        return new ParquetPageSource(parquetReader, typesBuilder.build(), fieldsBuilder.build(), namesBuilder.build(), hiveFileContext.getStats());
    } catch (Exception e) {
        try {
            if (dataSource != null) {
                dataSource.close();
            }
        } catch (IOException ignored) {
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        if (e instanceof ParquetCorruptionException) {
            throw new PrestoException(HIVE_BAD_DATA, e);
        }
        if (e instanceof AccessControlException) {
            throw new PrestoException(PERMISSION_DENIED, e.getMessage(), e);
        }
        if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) {
            throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e);
        }
        String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e.getClass().getSimpleName().equals("BlockMissingException")) {
            throw new PrestoException(HIVE_MISSING_DATA, message, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
    }
}

Example 74 with ParquetMetadata

use of org.apache.parquet.hadoop.metadata.ParquetMetadata in project presto by prestodb.

the class MetadataReader method readFooter.

public static ParquetFileMetadata readFooter(ParquetDataSource parquetDataSource, long fileSize) throws IOException {
    // Parquet File Layout:
    // 
    // MAGIC
    // variable: Data
    // variable: Metadata
    // 4 bytes: MetadataLength
    // MAGIC
    validateParquet(fileSize >= MAGIC.length() + POST_SCRIPT_SIZE, "%s is not a valid Parquet File", parquetDataSource.getId());
    // EXPECTED_FOOTER_SIZE is an int, so this will never fail
    byte[] buffer = new byte[toIntExact(min(fileSize, EXPECTED_FOOTER_SIZE))];
    parquetDataSource.readFully(fileSize - buffer.length, buffer);
    Slice tailSlice = wrappedBuffer(buffer);
    Slice magic = tailSlice.slice(tailSlice.length() - MAGIC.length(), MAGIC.length());
    if (!MAGIC.equals(magic)) {
        throw new ParquetCorruptionException(format("Not valid Parquet file: %s expected magic number: %s got: %s", parquetDataSource.getId(), Arrays.toString(MAGIC.getBytes()), Arrays.toString(magic.getBytes())));
    }
    int metadataLength = tailSlice.getInt(tailSlice.length() - POST_SCRIPT_SIZE);
    int completeFooterSize = metadataLength + POST_SCRIPT_SIZE;
    long metadataFileOffset = fileSize - completeFooterSize;
    validateParquet(metadataFileOffset >= MAGIC.length() && metadataFileOffset + POST_SCRIPT_SIZE < fileSize, "Corrupted Parquet file: %s metadata index: %s out of range", parquetDataSource.getId(), metadataFileOffset);
    // Ensure the slice covers the entire metadata range
    if (tailSlice.length() < completeFooterSize) {
        byte[] footerBuffer = new byte[completeFooterSize];
        parquetDataSource.readFully(metadataFileOffset, footerBuffer, 0, footerBuffer.length - tailSlice.length());
        // Copy the previous slice contents into the new buffer
        tailSlice.getBytes(0, footerBuffer, footerBuffer.length - tailSlice.length(), tailSlice.length());
        tailSlice = wrappedBuffer(footerBuffer, 0, footerBuffer.length);
    }
    FileMetaData fileMetaData = readFileMetaData(tailSlice.slice(tailSlice.length() - completeFooterSize, metadataLength).getInput());
    List<SchemaElement> schema = fileMetaData.getSchema();
    validateParquet(!schema.isEmpty(), "Empty Parquet schema in file: %s", parquetDataSource.getId());
    MessageType messageType = readParquetSchema(schema);
    List<BlockMetaData> blocks = new ArrayList<>();
    List<RowGroup> rowGroups = fileMetaData.getRow_groups();
    if (rowGroups != null) {
        for (RowGroup rowGroup : rowGroups) {
            BlockMetaData blockMetaData = new BlockMetaData();
            blockMetaData.setRowCount(rowGroup.getNum_rows());
            blockMetaData.setTotalByteSize(rowGroup.getTotal_byte_size());
            List<ColumnChunk> columns = rowGroup.getColumns();
            validateParquet(!columns.isEmpty(), "No columns in row group: %s", rowGroup);
            String filePath = columns.get(0).getFile_path();
            for (ColumnChunk columnChunk : columns) {
                validateParquet((filePath == null && columnChunk.getFile_path() == null) || (filePath != null && filePath.equals(columnChunk.getFile_path())), "all column chunks of the same row group must be in the same file");
                ColumnMetaData metaData = columnChunk.meta_data;
                String[] path = metaData.path_in_schema.stream().map(value -> value.toLowerCase(Locale.ENGLISH)).toArray(String[]::new);
                ColumnPath columnPath = ColumnPath.get(path);
                PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType();
                PrimitiveTypeName primitiveTypeName = primitiveType.getPrimitiveTypeName();
                ColumnChunkMetaData column = ColumnChunkMetaData.get(columnPath, primitiveType, CompressionCodecName.fromParquet(metaData.codec), PARQUET_METADATA_CONVERTER.convertEncodingStats(metaData.encoding_stats), readEncodings(metaData.encodings), readStats(metaData.statistics, primitiveTypeName), metaData.data_page_offset, metaData.dictionary_page_offset, metaData.num_values, metaData.total_compressed_size, metaData.total_uncompressed_size);
                column.setColumnIndexReference(toColumnIndexReference(columnChunk));
                column.setOffsetIndexReference(toOffsetIndexReference(columnChunk));
                blockMetaData.addColumn(column);
            }
            blockMetaData.setPath(filePath);
            blocks.add(blockMetaData);
        }
    }
    Map<String, String> keyValueMetaData = new HashMap<>();
    List<KeyValue> keyValueList = fileMetaData.getKey_value_metadata();
    if (keyValueList != null) {
        for (KeyValue keyValue : keyValueList) {
            keyValueMetaData.put(keyValue.key, keyValue.value);
        }
    }
    ParquetMetadata parquetMetadata = new ParquetMetadata(new org.apache.parquet.hadoop.metadata.FileMetaData(messageType, keyValueMetaData, fileMetaData.getCreated_by()), blocks);
    return new ParquetFileMetadata(parquetMetadata, toIntExact(metadataLength));
}

Example 75 with ParquetMetadata

use of org.apache.parquet.hadoop.metadata.ParquetMetadata in project flink by apache.

the class ParquetVectorizedInputFormat method createReader.

@Override
public ParquetReader createReader(final Configuration config, final SplitT split) throws IOException {
    final Path filePath = split.path();
    final long splitOffset = split.offset();
    final long splitLength = split.length();
    org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(filePath.toUri());
    ParquetMetadata footer = readFooter(hadoopConfig.conf(), hadoopPath, range(splitOffset, splitOffset + splitLength));
    MessageType fileSchema = footer.getFileMetaData().getSchema();
    FilterCompat.Filter filter = getFilter(hadoopConfig.conf());
    List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema);
    MessageType requestedSchema = clipParquetSchema(fileSchema);
    ParquetFileReader reader = new ParquetFileReader(hadoopConfig.conf(), footer.getFileMetaData(), hadoopPath, blocks, requestedSchema.getColumns());
    long totalRowCount = 0;
    for (BlockMetaData block : blocks) {
        totalRowCount += block.getRowCount();
    }
    checkSchema(fileSchema, requestedSchema);
    final Pool<ParquetReaderBatch<T>> poolOfBatches = createPoolOfBatches(split, requestedSchema, numBatchesToCirculate(config));
    return new ParquetReader(reader, requestedSchema, totalRowCount, poolOfBatches);
}