Examples with ColumnStatistics com.facebook.presto.orc.metadata.ColumnStatistics used on opensource projects

Example 1 with ColumnStatistics

use of com.facebook.presto.orc.metadata.ColumnStatistics in project presto by prestodb.

the class StripeReader method selectRowGroups.

private Set<Integer> selectRowGroups(StripeInformation stripe, Map<Integer, List<RowGroupIndex>> columnIndexes) throws IOException {
    int rowsInStripe = toIntExact(stripe.getNumberOfRows());
    int groupsInStripe = ceil(rowsInStripe, rowsInRowGroup);
    ImmutableSet.Builder<Integer> selectedRowGroups = ImmutableSet.builder();
    int remainingRows = rowsInStripe;
    for (int rowGroup = 0; rowGroup < groupsInStripe; ++rowGroup) {
        int rows = Math.min(remainingRows, rowsInRowGroup);
        Map<Integer, ColumnStatistics> statistics = getRowGroupStatistics(types.get(0), columnIndexes, rowGroup);
        if (predicate.matches(rows, statistics)) {
            selectedRowGroups.add(rowGroup);
        }
        remainingRows -= rows;
    }
    return selectedRowGroups.build();
}

Example 2 with ColumnStatistics

use of com.facebook.presto.orc.metadata.ColumnStatistics in project presto by prestodb.

the class StripeReader method getRowGroupStatistics.

private static Map<Integer, ColumnStatistics> getRowGroupStatistics(OrcType rootStructType, Map<Integer, List<RowGroupIndex>> columnIndexes, int rowGroup) {
    requireNonNull(rootStructType, "rootStructType is null");
    checkArgument(rootStructType.getOrcTypeKind() == OrcTypeKind.STRUCT);
    requireNonNull(columnIndexes, "columnIndexes is null");
    checkArgument(rowGroup >= 0, "rowGroup is negative");
    ImmutableMap.Builder<Integer, ColumnStatistics> statistics = ImmutableMap.builder();
    for (int ordinal = 0; ordinal < rootStructType.getFieldCount(); ordinal++) {
        List<RowGroupIndex> rowGroupIndexes = columnIndexes.get(rootStructType.getFieldTypeIndex(ordinal));
        if (rowGroupIndexes != null) {
            statistics.put(ordinal, rowGroupIndexes.get(rowGroup).getColumnStatistics());
        }
    }
    return statistics.build();
}

Example 3 with ColumnStatistics

use of com.facebook.presto.orc.metadata.ColumnStatistics in project presto by prestodb.

the class StripeReader method readColumnIndexes.

private Map<Integer, List<RowGroupIndex>> readColumnIndexes(Map<StreamId, Stream> streams, Map<StreamId, OrcInputStream> streamsData, Map<Integer, List<HiveBloomFilter>> bloomFilterIndexes) throws IOException {
    ImmutableMap.Builder<Integer, List<RowGroupIndex>> columnIndexes = ImmutableMap.builder();
    for (Entry<StreamId, Stream> entry : streams.entrySet()) {
        Stream stream = entry.getValue();
        if (stream.getStreamKind() == ROW_INDEX) {
            OrcInputStream inputStream = streamsData.get(entry.getKey());
            List<HiveBloomFilter> bloomFilters = bloomFilterIndexes.get(stream.getColumn());
            List<RowGroupIndex> rowGroupIndexes = metadataReader.readRowIndexes(hiveWriterVersion, inputStream);
            if (bloomFilters != null && !bloomFilters.isEmpty()) {
                ImmutableList.Builder<RowGroupIndex> newRowGroupIndexes = ImmutableList.builder();
                for (int i = 0; i < rowGroupIndexes.size(); i++) {
                    RowGroupIndex rowGroupIndex = rowGroupIndexes.get(i);
                    ColumnStatistics columnStatistics = rowGroupIndex.getColumnStatistics().withBloomFilter(bloomFilters.get(i));
                    newRowGroupIndexes.add(new RowGroupIndex(rowGroupIndex.getPositions(), columnStatistics));
                }
                rowGroupIndexes = newRowGroupIndexes.build();
            }
            columnIndexes.put(stream.getColumn(), rowGroupIndexes);
        }
    }
    return columnIndexes.build();
}

Example 4 with ColumnStatistics

use of com.facebook.presto.orc.metadata.ColumnStatistics in project presto by prestodb.

the class OrcRecordReader method getStatisticsByColumnOrdinal.

private static Map<Integer, ColumnStatistics> getStatisticsByColumnOrdinal(OrcType rootStructType, List<ColumnStatistics> fileStats) {
    requireNonNull(rootStructType, "rootStructType is null");
    checkArgument(rootStructType.getOrcTypeKind() == OrcTypeKind.STRUCT);
    requireNonNull(fileStats, "fileStats is null");
    ImmutableMap.Builder<Integer, ColumnStatistics> statistics = ImmutableMap.builder();
    for (int ordinal = 0; ordinal < rootStructType.getFieldCount(); ordinal++) {
        ColumnStatistics element = fileStats.get(rootStructType.getFieldTypeIndex(ordinal));
        if (element != null) {
            statistics.put(ordinal, element);
        }
    }
    return statistics.build();
}