Examples with ColumnStats com.facebook.presto.raptor.metadata.ColumnStats used on opensource projects

Example 6 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class TestShardOrganizerUtil method getShardIndexInfo.

private static List<ShardIndexInfo> getShardIndexInfo(Table tableInfo, List<ShardInfo> shards, TableColumn temporalColumn, Optional<List<TableColumn>> sortColumns) {
    long tableId = tableInfo.getTableId();
    Type temporalType = temporalColumn.getDataType();
    ImmutableList.Builder<ShardIndexInfo> builder = ImmutableList.builder();
    for (ShardInfo shard : shards) {
        ColumnStats temporalColumnStats = shard.getColumnStats().stream().filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()).findFirst().get();
        if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) {
            continue;
        }
        Optional<ShardRange> sortRange = Optional.empty();
        if (sortColumns.isPresent()) {
            Map<Long, ColumnStats> columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId);
            ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> minBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> maxBuilder = ImmutableList.builder();
            boolean isShardEligible = true;
            for (TableColumn sortColumn : sortColumns.get()) {
                ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId());
                typesBuilder.add(sortColumn.getDataType());
                if (columnStats.getMin() == null || columnStats.getMax() == null) {
                    isShardEligible = false;
                    break;
                }
                minBuilder.add(columnStats.getMin());
                maxBuilder.add(columnStats.getMax());
            }
            if (!isShardEligible) {
                continue;
            }
            List<Type> types = typesBuilder.build();
            List<Object> minValues = minBuilder.build();
            List<Object> maxValues = maxBuilder.build();
            sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues)));
        }
        builder.add(new ShardIndexInfo(tableId, OptionalInt.empty(), shard.getShardUuid(), shard.getRowCount(), shard.getUncompressedSize(), sortRange, Optional.of(ShardRange.of(new Tuple(temporalType, temporalColumnStats.getMin()), new Tuple(temporalType, temporalColumnStats.getMax())))));
    }
    return builder.build();
}

Example 7 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class TestOrcStorageManager method assertColumnStats.

private static void assertColumnStats(List<ColumnStats> list, long columnId, Object min, Object max) {
    for (ColumnStats stats : list) {
        if (stats.getColumnId() == columnId) {
            assertEquals(stats.getMin(), min);
            assertEquals(stats.getMax(), max);
            return;
        }
    }
    fail(format("no stats for column: %s: %s", columnId, list));
}

Example 8 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class OrcStorageManager method computeShardStats.

private List<ColumnStats> computeShardStats(File file) {
    try (OrcDataSource dataSource = fileOrcDataSource(defaultReaderAttributes, file)) {
        OrcReader reader = new OrcReader(dataSource, new OrcMetadataReader(), defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getMaxReadSize());
        ImmutableList.Builder<ColumnStats> list = ImmutableList.builder();
        for (ColumnInfo info : getColumnInfo(reader)) {
            computeColumnStats(reader, info.getColumnId(), info.getType()).ifPresent(list::add);
        }
        return list.build();
    } catch (IOException e) {
        throw new PrestoException(RAPTOR_ERROR, "Failed to read file: " + file, e);
    }
}