Examples with StreamCheckpoint com.facebook.presto.orc.checkpoint.StreamCheckpoint used on opensource projects

Example 6 with StreamCheckpoint

use of com.facebook.presto.orc.checkpoint.StreamCheckpoint in project presto by prestodb.

the class StripeReader method createDictionaryStreamSources.

public InputStreamSources createDictionaryStreamSources(Map<StreamId, Stream> streams, Map<StreamId, ValueInputStream<?>> valueStreams, Map<Integer, ColumnEncoding> columnEncodings) {
    ImmutableMap.Builder<StreamId, InputStreamSource<?>> dictionaryStreamBuilder = ImmutableMap.builder();
    for (Entry<StreamId, Stream> entry : streams.entrySet()) {
        StreamId streamId = entry.getKey();
        Stream stream = entry.getValue();
        int column = stream.getColumn();
        // only process dictionary streams
        ColumnEncodingKind columnEncoding = columnEncodings.get(column).getColumnEncoding(stream.getSequence()).getColumnEncodingKind();
        if (!isDictionary(stream, columnEncoding)) {
            continue;
        }
        // skip streams without data
        ValueInputStream<?> valueStream = valueStreams.get(streamId);
        if (valueStream == null) {
            continue;
        }
        OrcTypeKind columnType = types.get(stream.getColumn()).getOrcTypeKind();
        StreamCheckpoint streamCheckpoint = getDictionaryStreamCheckpoint(streamId, columnType, columnEncoding);
        InputStreamSource<?> streamSource = createCheckpointStreamSource(valueStream, streamCheckpoint);
        dictionaryStreamBuilder.put(streamId, streamSource);
    }
    return new InputStreamSources(dictionaryStreamBuilder.build());
}

Example 7 with StreamCheckpoint

use of com.facebook.presto.orc.checkpoint.StreamCheckpoint in project presto by prestodb.

the class StripeReader method createRowGroup.

@VisibleForTesting
static RowGroup createRowGroup(int groupId, long rowsInStripe, long rowsInRowGroup, Map<StreamId, List<RowGroupIndex>> columnIndexes, Map<StreamId, ValueInputStream<?>> valueStreams, Map<StreamId, StreamCheckpoint> checkpoints) {
    long totalRowGroupBytes = columnIndexes.values().stream().mapToLong(e -> e.get(groupId).getColumnStatistics().getTotalValueSizeInBytes()).sum();
    long rowOffset = multiplyExact(groupId, rowsInRowGroup);
    int rowCount = toIntExact(Math.min(rowsInStripe - rowOffset, rowsInRowGroup));
    ImmutableMap.Builder<StreamId, InputStreamSource<?>> builder = ImmutableMap.builder();
    for (Entry<StreamId, StreamCheckpoint> entry : checkpoints.entrySet()) {
        StreamId streamId = entry.getKey();
        StreamCheckpoint checkpoint = entry.getValue();
        // skip streams without data
        ValueInputStream<?> valueStream = valueStreams.get(streamId);
        if (valueStream == null) {
            continue;
        }
        builder.put(streamId, createCheckpointStreamSource(valueStream, checkpoint));
    }
    InputStreamSources rowGroupStreams = new InputStreamSources(builder.build());
    return new RowGroup(groupId, rowOffset, rowCount, totalRowGroupBytes, rowGroupStreams);
}