Examples with StreamDataOutput com.facebook.presto.orc.stream.StreamDataOutput used on opensource projects

Example 6 with StreamDataOutput

use of com.facebook.presto.orc.stream.StreamDataOutput in project presto by prestodb.

the class TestLongDictionaryProvider method createSliceOutput.

private DynamicSliceOutput createSliceOutput(StreamId streamId, long[] data) {
    LongOutputStreamDwrf outputStream = new LongOutputStreamDwrf(getColumnWriterOptions(), Optional.empty(), true, DICTIONARY_DATA);
    for (long val : data) {
        outputStream.writeLong(val);
    }
    outputStream.close();
    DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1000);
    StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(streamId.getColumn());
    streamDataOutput.writeData(sliceOutput);
    return sliceOutput;
}

Example 7 with StreamDataOutput

use of com.facebook.presto.orc.stream.StreamDataOutput in project presto by prestodb.

the class TestStreamLayout method testByStreamSize.

@Test
public void testByStreamSize() {
    List<StreamDataOutput> streams = new ArrayList<>();
    int length = 10_000;
    for (int i = 0; i < 10; i++) {
        streams.add(createStream(i, StreamKind.PRESENT, length - i));
        streams.add(createStream(i, StreamKind.DATA, length - 100 - i));
    }
    Collections.shuffle(streams);
    new ByStreamSize().reorder(streams);
    assertEquals(streams.size(), 20);
    Iterator<StreamDataOutput> iterator = streams.iterator();
    for (int i = 9; i >= 0; i--) {
        verifyStream(iterator.next().getStream(), i, StreamKind.DATA, length - 100 - i);
    }
    for (int i = 9; i >= 0; i--) {
        verifyStream(iterator.next().getStream(), i, StreamKind.PRESENT, length - i);
    }
    assertFalse(iterator.hasNext());
}

Example 8 with StreamDataOutput

use of com.facebook.presto.orc.stream.StreamDataOutput in project presto by prestodb.

the class FloatColumnWriter method getIndexStreams.

@Override
public List<StreamDataOutput> getIndexStreams() throws IOException {
    checkState(closed);
    ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder();
    List<FloatStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints();
    Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints();
    for (int i = 0; i < rowGroupColumnStatistics.size(); i++) {
        int groupId = i;
        ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
        FloatStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
        Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
        List<Integer> positions = createFloatColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
        rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
    }
    Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build());
    Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false);
    return ImmutableList.of(new StreamDataOutput(slice, stream));
}

Example 9 with StreamDataOutput

use of com.facebook.presto.orc.stream.StreamDataOutput in project presto by prestodb.

the class StructColumnWriter method getIndexStreams.

@Override
public List<StreamDataOutput> getIndexStreams() throws IOException {
    checkState(closed);
    ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder();
    Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints();
    for (int i = 0; i < rowGroupColumnStatistics.size(); i++) {
        int groupId = i;
        ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
        Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
        List<Integer> positions = createStructColumnPositionList(compressed, presentCheckpoint);
        rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
    }
    Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build());
    Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false);
    ImmutableList.Builder<StreamDataOutput> indexStreams = ImmutableList.builder();
    indexStreams.add(new StreamDataOutput(slice, stream));
    for (ColumnWriter structField : structFields) {
        indexStreams.addAll(structField.getIndexStreams());
    }
    return indexStreams.build();
}

Example 10 with StreamDataOutput

use of com.facebook.presto.orc.stream.StreamDataOutput in project presto by prestodb.

the class LongColumnWriter method getIndexStreams.

@Override
public List<StreamDataOutput> getIndexStreams() throws IOException {
    checkState(closed);
    ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder();
    List<LongStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints();
    Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints();
    for (int i = 0; i < rowGroupColumnStatistics.size(); i++) {
        int groupId = i;
        ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
        LongStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
        Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
        List<Integer> positions = createLongColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
        rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
    }
    Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build());
    Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false);
    return ImmutableList.of(new StreamDataOutput(slice, stream));
}