Search in sources :

Example 1 with FileMetadata

use of org.apache.drill.metastore.metadata.FileMetadata in project drill by apache.

the class AbstractParquetGroupScan method applyLimit.

// filter push down methods block end
// limit push down methods start
@Override
public GroupScan applyLimit(int maxRecords) {
    // Make sure it request at least 1 row -> 1 rowGroup.
    maxRecords = Math.max(maxRecords, 1);
    if (getTableMetadata() != null) {
        long tableRowCount = TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata());
        if (tableRowCount == Statistic.NO_COLUMN_STATS || tableRowCount <= maxRecords) {
            logger.debug("limit push down does not apply, since total number of rows [{}] is less or equal to the required [{}].", tableRowCount, maxRecords);
            return null;
        }
    }
    List<RowGroupMetadata> qualifiedRowGroups = limitMetadata(getRowGroupsMetadata().values(), maxRecords);
    if (qualifiedRowGroups == null || getRowGroupsMetadata().size() == qualifiedRowGroups.size()) {
        logger.debug("limit push down does not apply, since number of row groups was not reduced.");
        return null;
    }
    Map<Path, FileMetadata> filesMetadata = getFilesMetadata();
    Map<Path, FileMetadata> qualifiedFiles = qualifiedRowGroups.stream().map(rowGroup -> filesMetadata.get(rowGroup.getPath())).filter(Objects::nonNull).collect(Collectors.toMap(FileMetadata::getPath, Function.identity()));
    Multimap<Path, RowGroupMetadata> prunedRowGroups = LinkedListMultimap.create();
    for (RowGroupMetadata qualifiedRowGroup : qualifiedRowGroups) {
        prunedRowGroups.put(qualifiedRowGroup.getPath(), qualifiedRowGroup);
    }
    return getFilterer().rowGroups(prunedRowGroups).table(tableMetadata).partitions(partitions).segments(segments).files(qualifiedFiles).nonInterestingColumns(nonInterestingColumnsMetadata).matching(matchAllMetadata).build();
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) ReadEntryWithPath(org.apache.drill.exec.store.dfs.ReadEntryWithPath) FileMetadata(org.apache.drill.metastore.metadata.FileMetadata) RowGroupMetadata(org.apache.drill.metastore.metadata.RowGroupMetadata)

Example 2 with FileMetadata

use of org.apache.drill.metastore.metadata.FileMetadata in project drill by apache.

the class ParquetTableMetadataUtils method getPartitionMetadata.

/**
 * Returns {@link PartitionMetadata} instance received by merging specified {@link FileMetadata} list.
 *
 * @param partitionColumn partition column
 * @param files           list of files to be merged
 * @return {@link PartitionMetadata} instance
 */
public static PartitionMetadata getPartitionMetadata(SchemaPath partitionColumn, List<FileMetadata> files) {
    Set<Path> locations = new HashSet<>();
    Set<SchemaPath> columns = new HashSet<>();
    for (FileMetadata file : files) {
        columns.addAll(file.getColumnsStatistics().keySet());
        locations.add(file.getPath());
    }
    FileMetadata fileMetadata = files.iterator().next();
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.PARTITION).build();
    return PartitionMetadata.builder().tableInfo(fileMetadata.getTableInfo()).metadataInfo(metadataInfo).column(partitionColumn).schema(fileMetadata.getSchema()).columnsStatistics(TableMetadataUtils.mergeColumnsStatistics(files, columns, PARQUET_COLUMN_STATISTICS)).metadataStatistics(Collections.singletonList(new StatisticsHolder<>(TableStatisticsKind.ROW_COUNT.mergeStatistics(files), TableStatisticsKind.ROW_COUNT))).partitionValues(Collections.emptyList()).locations(locations).build();
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) MetadataInfo(org.apache.drill.metastore.metadata.MetadataInfo) SchemaPath(org.apache.drill.common.expression.SchemaPath) FileMetadata(org.apache.drill.metastore.metadata.FileMetadata) HashSet(java.util.HashSet)

Example 3 with FileMetadata

use of org.apache.drill.metastore.metadata.FileMetadata in project drill by apache.

the class BaseParquetMetadataProvider method getSegmentsMetadataMap.

@SuppressWarnings("unused")
@Override
public Map<Path, SegmentMetadata> getSegmentsMetadataMap() {
    if (segments == null) {
        if (entries.isEmpty() || !collectMetadata) {
            return Collections.emptyMap();
        }
        segments = new LinkedHashMap<>();
        Path fileLocation = getFilesMetadataMap().values().iterator().next().getPath();
        int levelsCount = fileLocation.depth() - tableLocation.depth();
        Map<Path, FileMetadata> filesMetadata = getFilesMetadataMap();
        int segmentsIndex = levelsCount - 1;
        Map<Path, SegmentMetadata> segmentMetadata = getSegmentsForMetadata(filesMetadata, SchemaPath.getSimplePath(MetadataInfo.DEFAULT_COLUMN_PREFIX + segmentsIndex));
        segments.putAll(segmentMetadata);
        for (int i = segmentsIndex - 1; i >= 0; i--) {
            String segmentColumn = MetadataInfo.DEFAULT_COLUMN_PREFIX + i;
            segmentMetadata = getMetadataForSegments(segmentMetadata, SchemaPath.getSimplePath(segmentColumn));
            segments.putAll(segmentMetadata);
        }
    }
    return segments;
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) ReadEntryWithPath(org.apache.drill.exec.store.dfs.ReadEntryWithPath) SegmentMetadata(org.apache.drill.metastore.metadata.SegmentMetadata) FileMetadata(org.apache.drill.metastore.metadata.FileMetadata)

Example 4 with FileMetadata

use of org.apache.drill.metastore.metadata.FileMetadata in project drill by apache.

the class BasicTablesTransformer method all.

public static MetadataHolder all(List<TableMetadataUnit> units) {
    List<BaseTableMetadata> tables = new ArrayList<>();
    List<SegmentMetadata> segments = new ArrayList<>();
    List<FileMetadata> files = new ArrayList<>();
    List<RowGroupMetadata> rowGroups = new ArrayList<>();
    List<PartitionMetadata> partitions = new ArrayList<>();
    for (TableMetadataUnit unit : units) {
        MetadataType metadataType = MetadataType.fromValue(unit.metadataType());
        if (metadataType == null) {
            continue;
        }
        switch(metadataType) {
            case TABLE:
                tables.add(BaseTableMetadata.builder().metadataUnit(unit).build());
                break;
            case SEGMENT:
                segments.add(SegmentMetadata.builder().metadataUnit(unit).build());
                break;
            case FILE:
                files.add(FileMetadata.builder().metadataUnit(unit).build());
                break;
            case ROW_GROUP:
                rowGroups.add(RowGroupMetadata.builder().metadataUnit(unit).build());
                break;
            case PARTITION:
                partitions.add(PartitionMetadata.builder().metadataUnit(unit).build());
                break;
            default:
                // Ignore unsupported type
                break;
        }
    }
    return new MetadataHolder(tables, segments, files, rowGroups, partitions);
}
Also used : ArrayList(java.util.ArrayList) FileMetadata(org.apache.drill.metastore.metadata.FileMetadata) MetadataType(org.apache.drill.metastore.metadata.MetadataType) RowGroupMetadata(org.apache.drill.metastore.metadata.RowGroupMetadata) SegmentMetadata(org.apache.drill.metastore.metadata.SegmentMetadata) BaseTableMetadata(org.apache.drill.metastore.metadata.BaseTableMetadata) PartitionMetadata(org.apache.drill.metastore.metadata.PartitionMetadata)

Example 5 with FileMetadata

use of org.apache.drill.metastore.metadata.FileMetadata in project drill by apache.

the class MetadataControllerBatch method getFileMetadata.

private FileMetadata getFileMetadata(TupleReader reader, List<StatisticsHolder<?>> metadataStatistics, Map<SchemaPath, ColumnStatistics<?>> columnStatistics, int nestingLevel) {
    List<String> segmentColumns = popConfig.getContext().segmentColumns();
    String segmentKey = segmentColumns.size() > 0 ? reader.column(segmentColumns.iterator().next()).scalar().getString() : MetadataInfo.DEFAULT_SEGMENT_KEY;
    List<String> partitionValues = segmentColumns.stream().limit(nestingLevel - 1).map(columnName -> reader.column(columnName).scalar().getString()).collect(Collectors.toList());
    Path path = new Path(reader.column(MetastoreAnalyzeConstants.LOCATION_FIELD).scalar().getString());
    String metadataIdentifier = MetadataIdentifierUtils.getFileMetadataIdentifier(partitionValues, path);
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.FILE).key(segmentKey).identifier(StringUtils.defaultIfEmpty(metadataIdentifier, null)).build();
    return FileMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).columnsStatistics(columnStatistics).metadataStatistics(metadataStatistics).path(path).lastModifiedTime(Long.parseLong(reader.column(columnNamesOptions.lastModifiedTime()).scalar().getString())).schema(TupleMetadata.of(reader.column(MetastoreAnalyzeConstants.SCHEMA_FIELD).scalar().getString())).build();
}
Also used : MetadataType(org.apache.drill.metastore.metadata.MetadataType) BaseStatisticsKind(org.apache.drill.metastore.statistics.BaseStatisticsKind) MetastoreColumn(org.apache.drill.metastore.MetastoreColumn) UserException(org.apache.drill.common.exceptions.UserException) LoggerFactory(org.slf4j.LoggerFactory) Types(org.apache.drill.common.types.Types) MetadataInfo(org.apache.drill.metastore.metadata.MetadataInfo) RowSetReader(org.apache.drill.exec.physical.rowSet.RowSetReader) VectorContainer(org.apache.drill.exec.record.VectorContainer) ColumnStatistics(org.apache.drill.metastore.statistics.ColumnStatistics) ArrayListMultimap(org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap) StringUtils(org.apache.commons.lang3.StringUtils) ArrayReader(org.apache.drill.exec.vector.accessor.ArrayReader) BaseTableMetadata(org.apache.drill.metastore.metadata.BaseTableMetadata) StatisticsRecordWriterImpl(org.apache.drill.exec.store.StatisticsRecordWriterImpl) PartitionMetadata(org.apache.drill.metastore.metadata.PartitionMetadata) Map(java.util.Map) FieldConverter(org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter) Path(org.apache.hadoop.fs.Path) BatchSchema(org.apache.drill.exec.record.BatchSchema) ColumnMetadata(org.apache.drill.exec.record.metadata.ColumnMetadata) Multimap(org.apache.drill.shaded.guava.com.google.common.collect.Multimap) ColumnStatisticsKind(org.apache.drill.metastore.statistics.ColumnStatisticsKind) Delete(org.apache.drill.metastore.operate.Delete) TableMetadataUnit(org.apache.drill.metastore.components.tables.TableMetadataUnit) SegmentMetadata(org.apache.drill.metastore.metadata.SegmentMetadata) SchemaPath(org.apache.drill.common.expression.SchemaPath) RecordBatch(org.apache.drill.exec.record.RecordBatch) Set(java.util.Set) MetastoreAnalyzeConstants(org.apache.drill.exec.metastore.analyze.MetastoreAnalyzeConstants) Collectors(java.util.stream.Collectors) TupleMetadata(org.apache.drill.exec.record.metadata.TupleMetadata) FieldReader(org.apache.drill.exec.vector.complex.reader.FieldReader) TypeProtos(org.apache.drill.common.types.TypeProtos) List(java.util.List) AbstractBinaryRecordBatch(org.apache.drill.exec.record.AbstractBinaryRecordBatch) Preconditions(org.apache.drill.shaded.guava.com.google.common.base.Preconditions) ObjectReader(org.apache.drill.exec.vector.accessor.ObjectReader) TableInfo(org.apache.drill.metastore.metadata.TableInfo) MetadataIdentifierUtils(org.apache.drill.exec.metastore.analyze.MetadataIdentifierUtils) TupleReader(org.apache.drill.exec.vector.accessor.TupleReader) Modify(org.apache.drill.metastore.operate.Modify) MetadataControllerContext(org.apache.drill.exec.metastore.analyze.MetadataControllerContext) HashMap(java.util.HashMap) BitVector(org.apache.drill.exec.vector.BitVector) Function(java.util.function.Function) VectorWrapper(org.apache.drill.exec.record.VectorWrapper) ArrayList(java.util.ArrayList) ColumnNamesOptions(org.apache.drill.exec.metastore.ColumnNamesOptions) HashSet(java.util.HashSet) OutOfMemoryException(org.apache.drill.exec.exception.OutOfMemoryException) DirectRowSet(org.apache.drill.exec.physical.rowSet.DirectRowSet) DrillStatsTable(org.apache.drill.exec.planner.common.DrillStatsTable) WriterPrel(org.apache.drill.exec.planner.physical.WriterPrel) TableStatisticsKind(org.apache.drill.metastore.statistics.TableStatisticsKind) FragmentContext(org.apache.drill.exec.ops.FragmentContext) FileMetadata(org.apache.drill.metastore.metadata.FileMetadata) StatisticsRecordCollector(org.apache.drill.exec.store.StatisticsRecordCollector) BaseMetadata(org.apache.drill.metastore.metadata.BaseMetadata) Logger(org.slf4j.Logger) ExactStatisticsConstants(org.apache.drill.metastore.statistics.ExactStatisticsConstants) RowGroupMetadata(org.apache.drill.metastore.metadata.RowGroupMetadata) StatisticsKind(org.apache.drill.metastore.statistics.StatisticsKind) IOException(java.io.IOException) FilterExpression(org.apache.drill.metastore.expressions.FilterExpression) StatisticsCollectorImpl(org.apache.drill.exec.store.easy.json.StatisticsCollectorImpl) PlannerSettings(org.apache.drill.exec.planner.physical.PlannerSettings) ParquetTableMetadataUtils(org.apache.drill.exec.store.parquet.ParquetTableMetadataUtils) VarCharVector(org.apache.drill.exec.vector.VarCharVector) StatisticsHolder(org.apache.drill.metastore.statistics.StatisticsHolder) MetadataControllerPOP(org.apache.drill.exec.physical.config.MetadataControllerPOP) Tables(org.apache.drill.metastore.components.tables.Tables) Collections(java.util.Collections) AnalyzeColumnUtils(org.apache.drill.exec.metastore.analyze.AnalyzeColumnUtils) MetastoreTableInfo(org.apache.drill.metastore.components.tables.MetastoreTableInfo) ObjectType(org.apache.drill.exec.vector.accessor.ObjectType) Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) MetadataInfo(org.apache.drill.metastore.metadata.MetadataInfo)

Aggregations

FileMetadata (org.apache.drill.metastore.metadata.FileMetadata)23 MetastoreTest (org.apache.drill.categories.MetastoreTest)17 SchemaPath (org.apache.drill.common.expression.SchemaPath)17 BaseTableMetadata (org.apache.drill.metastore.metadata.BaseTableMetadata)17 Path (org.apache.hadoop.fs.Path)17 Test (org.junit.Test)17 TableInfo (org.apache.drill.metastore.metadata.TableInfo)16 MetastoreTableInfo (org.apache.drill.metastore.components.tables.MetastoreTableInfo)15 File (java.io.File)14 SlowTest (org.apache.drill.categories.SlowTest)14 ClusterTest (org.apache.drill.test.ClusterTest)14 ColumnStatistics (org.apache.drill.metastore.statistics.ColumnStatistics)13 RowGroupMetadata (org.apache.drill.metastore.metadata.RowGroupMetadata)12 SegmentMetadata (org.apache.drill.metastore.metadata.SegmentMetadata)12 TupleMetadata (org.apache.drill.exec.record.metadata.TupleMetadata)9 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)9 HashMap (java.util.HashMap)8 SchemaBuilder (org.apache.drill.exec.record.metadata.SchemaBuilder)8 StatisticsHolder (org.apache.drill.metastore.statistics.StatisticsHolder)8 MetadataInfo (org.apache.drill.metastore.metadata.MetadataInfo)7