Examples with StreamId com.facebook.presto.orc.StreamId used on opensource projects

Example 1 with StreamId

use of com.facebook.presto.orc.StreamId in project presto by prestodb.

the class Checkpoints method getStreamCheckpoints.

public static Map<StreamId, StreamCheckpoint> getStreamCheckpoints(Set<Integer> columns, List<OrcType> columnTypes, CompressionKind compressionKind, int rowGroupId, List<ColumnEncoding> columnEncodings, Map<StreamId, Stream> streams, Map<Integer, List<RowGroupIndex>> columnIndexes) throws InvalidCheckpointException {
    ImmutableSetMultimap.Builder<Integer, StreamKind> streamKindsBuilder = ImmutableSetMultimap.builder();
    for (Stream stream : streams.values()) {
        streamKindsBuilder.put(stream.getColumn(), stream.getStreamKind());
    }
    SetMultimap<Integer, StreamKind> streamKinds = streamKindsBuilder.build();
    ImmutableMap.Builder<StreamId, StreamCheckpoint> checkpoints = ImmutableMap.builder();
    for (int column : columns) {
        List<Integer> positionsList = columnIndexes.get(column).get(rowGroupId).getPositions();
        ColumnEncodingKind columnEncoding = columnEncodings.get(column).getColumnEncodingKind();
        OrcTypeKind columnType = columnTypes.get(column).getOrcTypeKind();
        Set<StreamKind> availableStreams = streamKinds.get(column);
        ColumnPositionsList columnPositionsList = new ColumnPositionsList(column, columnType, positionsList);
        switch(columnType) {
            case BOOLEAN:
                checkpoints.putAll(getBooleanColumnCheckpoints(column, compressionKind, availableStreams, columnPositionsList));
                break;
            case BYTE:
                checkpoints.putAll(getByteColumnCheckpoints(column, compressionKind, availableStreams, columnPositionsList));
                break;
            case SHORT:
            case INT:
            case LONG:
            case DATE:
                checkpoints.putAll(getLongColumnCheckpoints(column, columnEncoding, compressionKind, availableStreams, columnPositionsList));
                break;
            case FLOAT:
                checkpoints.putAll(getFloatColumnCheckpoints(column, compressionKind, availableStreams, columnPositionsList));
                break;
            case DOUBLE:
                checkpoints.putAll(getDoubleColumnCheckpoints(column, compressionKind, availableStreams, columnPositionsList));
                break;
            case TIMESTAMP:
                checkpoints.putAll(getTimestampColumnCheckpoints(column, columnEncoding, compressionKind, availableStreams, columnPositionsList));
                break;
            case BINARY:
            case STRING:
            case VARCHAR:
            case CHAR:
                checkpoints.putAll(getSliceColumnCheckpoints(column, columnEncoding, compressionKind, availableStreams, columnPositionsList));
                break;
            case LIST:
            case MAP:
                checkpoints.putAll(getListOrMapColumnCheckpoints(column, columnEncoding, compressionKind, availableStreams, columnPositionsList));
                break;
            case STRUCT:
                checkpoints.putAll(getStructColumnCheckpoints(column, compressionKind, availableStreams, columnPositionsList));
                break;
            case DECIMAL:
                checkpoints.putAll(getDecimalColumnCheckpoints(column, columnEncoding, compressionKind, availableStreams, columnPositionsList));
                break;
            case UNION:
                throw new IllegalArgumentException("Unsupported column type " + columnType);
        }
        // clear the extra offsets
        if (columnPositionsList.hasNextPosition() && !Iterables.all(positionsList, equalTo(0))) {
            throw new InvalidCheckpointException(format("Column %s, of type %s, contains %s offset positions, but only %s positions were consumed", column, columnType, positionsList.size(), columnPositionsList.getIndex()));
        }
    }
    return checkpoints.build();
}

Example 2 with StreamId

use of com.facebook.presto.orc.StreamId in project presto by prestodb.

the class StreamSources method getStreamSource.

@Nonnull
public <S extends ValueStream<?>> StreamSource<S> getStreamSource(StreamDescriptor streamDescriptor, StreamKind streamKind, Class<S> streamType) {
    requireNonNull(streamDescriptor, "streamDescriptor is null");
    requireNonNull(streamType, "streamType is null");
    StreamSource<?> streamSource = streamSources.get(new StreamId(streamDescriptor.getStreamId(), streamKind));
    if (streamSource == null) {
        streamSource = missingStreamSource(streamType);
    }
    checkArgument(streamType.isAssignableFrom(streamSource.getStreamType()), "%s must be of type %s, not %s", streamDescriptor, streamType.getName(), streamSource.getStreamType().getName());
    return (StreamSource<S>) streamSource;
}

Example 3 with StreamId

use of com.facebook.presto.orc.StreamId in project presto by prestodb.

the class Checkpoints method getStreamCheckpoints.

public static Map<StreamId, StreamCheckpoint> getStreamCheckpoints(Set<Integer> columns, List<OrcType> columnTypes, boolean compressed, int rowGroupId, Map<Integer, ColumnEncoding> columnEncodings, Map<StreamId, Stream> streams, Map<StreamId, List<RowGroupIndex>> columnIndexes) throws InvalidCheckpointException {
    // map from (column, sequence) to available StreamKind
    ImmutableSetMultimap.Builder<ColumnAndSequence, StreamKind> streamKindsBuilder = ImmutableSetMultimap.builder();
    for (Stream stream : streams.values()) {
        streamKindsBuilder.put(new ColumnAndSequence(stream.getColumn(), stream.getSequence()), stream.getStreamKind());
    }
    SetMultimap<ColumnAndSequence, StreamKind> streamKinds = streamKindsBuilder.build();
    ImmutableMap.Builder<StreamId, StreamCheckpoint> checkpoints = ImmutableMap.builder();
    for (Map.Entry<StreamId, List<RowGroupIndex>> entry : columnIndexes.entrySet()) {
        int column = entry.getKey().getColumn();
        if (!columns.contains(column)) {
            continue;
        }
        int sequence = entry.getKey().getSequence();
        List<Integer> positionsList = entry.getValue().get(rowGroupId).getPositions();
        ColumnEncodingKind columnEncoding = columnEncodings.get(column).getColumnEncoding(sequence).getColumnEncodingKind();
        OrcTypeKind columnType = columnTypes.get(column).getOrcTypeKind();
        Set<StreamKind> availableStreams = streamKinds.get(new ColumnAndSequence(column, sequence));
        ColumnPositionsList columnPositionsList = new ColumnPositionsList(column, sequence, columnType, positionsList);
        switch(columnType) {
            case BOOLEAN:
                checkpoints.putAll(getBooleanColumnCheckpoints(column, sequence, compressed, availableStreams, columnPositionsList));
                break;
            case BYTE:
                checkpoints.putAll(getByteColumnCheckpoints(column, sequence, compressed, availableStreams, columnPositionsList));
                break;
            case SHORT:
            case INT:
            case LONG:
            case DATE:
                checkpoints.putAll(getLongColumnCheckpoints(column, sequence, columnEncoding, compressed, availableStreams, columnPositionsList));
                break;
            case FLOAT:
                checkpoints.putAll(getFloatColumnCheckpoints(column, sequence, compressed, availableStreams, columnPositionsList));
                break;
            case DOUBLE:
                checkpoints.putAll(getDoubleColumnCheckpoints(column, sequence, compressed, availableStreams, columnPositionsList));
                break;
            case TIMESTAMP:
                checkpoints.putAll(getTimestampColumnCheckpoints(column, sequence, columnEncoding, compressed, availableStreams, columnPositionsList));
                break;
            case BINARY:
            case STRING:
            case VARCHAR:
            case CHAR:
                checkpoints.putAll(getSliceColumnCheckpoints(column, sequence, columnEncoding, compressed, availableStreams, columnPositionsList));
                break;
            case LIST:
            case MAP:
                checkpoints.putAll(getListOrMapColumnCheckpoints(column, sequence, columnEncoding, compressed, availableStreams, columnPositionsList));
                break;
            case STRUCT:
                checkpoints.putAll(getStructColumnCheckpoints(column, sequence, compressed, availableStreams, columnPositionsList));
                break;
            case DECIMAL:
                checkpoints.putAll(getDecimalColumnCheckpoints(column, sequence, columnEncoding, compressed, availableStreams, columnPositionsList));
                break;
            default:
                throw new IllegalArgumentException("Unsupported column type " + columnType);
        }
        // clear the extra offsets
        if (columnPositionsList.hasNextPosition() && !Iterables.all(positionsList, equalTo(0))) {
            throw new InvalidCheckpointException(format("Column %s, of type %s, contains %s offset positions, but only %s positions were consumed", column, columnType, positionsList.size(), columnPositionsList.getIndex()));
        }
    }
    return checkpoints.build();
}

Example 4 with StreamId

use of com.facebook.presto.orc.StreamId in project presto by prestodb.

the class InputStreamSources method getInputStreamSource.

public <S extends ValueInputStream<?>> InputStreamSource<S> getInputStreamSource(StreamDescriptor streamDescriptor, StreamKind streamKind, Class<S> streamType) {
    requireNonNull(streamDescriptor, "streamDescriptor is null");
    requireNonNull(streamType, "streamType is null");
    InputStreamSource<?> streamSource = streamSources.get(new StreamId(streamDescriptor.getStreamId(), streamDescriptor.getSequence(), streamKind));
    if (streamSource == null) {
        streamSource = missingStreamSource(streamType);
    }
    checkArgument(streamType.isAssignableFrom(streamSource.getStreamType()), "%s must be of type %s, not %s", streamDescriptor, streamType.getName(), streamSource.getStreamType().getName());
    return (InputStreamSource<S>) streamSource;
}