Examples with FixedLengthSliceInput io.airlift.slice.FixedLengthSliceInput used on opensource projects

Example 1 with FixedLengthSliceInput

use of io.airlift.slice.FixedLengthSliceInput in project presto by prestodb.

the class StripeReader method readDiskRanges.

public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AbstractAggregatedMemoryContext systemMemoryUsage) throws IOException {
    //
    // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation
    //
    // transform ranges to have an absolute offset in file
    ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder();
    for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) {
        DiskRange diskRange = entry.getValue();
        diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength()));
    }
    diskRanges = diskRangesBuilder.build();
    // read ranges
    Map<StreamId, FixedLengthSliceInput> streamsData = orcDataSource.readFully(diskRanges);
    // transform streams to OrcInputStream
    String sourceName = orcDataSource.toString();
    ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder();
    for (Entry<StreamId, FixedLengthSliceInput> entry : streamsData.entrySet()) {
        streamsBuilder.put(entry.getKey(), new OrcInputStream(sourceName, entry.getValue(), compressionKind, bufferSize, systemMemoryUsage));
    }
    return streamsBuilder.build();
}

Example 2 with FixedLengthSliceInput

use of io.airlift.slice.FixedLengthSliceInput in project presto by prestodb.

the class AbstractOrcDataSource method readLargeDiskRanges.

private <K> Map<K, OrcDataSourceInput> readLargeDiskRanges(Map<K, DiskRange> diskRanges) {
    if (diskRanges.isEmpty()) {
        return ImmutableMap.of();
    }
    ImmutableMap.Builder<K, OrcDataSourceInput> slices = ImmutableMap.builder();
    for (Entry<K, DiskRange> entry : diskRanges.entrySet()) {
        DiskRange diskRange = entry.getValue();
        int bufferSize = toIntExact(streamBufferSize.toBytes());
        FixedLengthSliceInput sliceInput = new LazySliceInput(diskRange.getLength(), new LazyChunkedSliceLoader(diskRange, bufferSize));
        slices.put(entry.getKey(), new OrcDataSourceInput(sliceInput, bufferSize));
    }
    return slices.build();
}

Example 3 with FixedLengthSliceInput

use of io.airlift.slice.FixedLengthSliceInput in project presto by prestodb.

the class AbstractOrcDataSource method readSmallDiskRanges.

private <K> Map<K, OrcDataSourceInput> readSmallDiskRanges(Map<K, DiskRange> diskRanges) throws IOException {
    if (diskRanges.isEmpty()) {
        return ImmutableMap.of();
    }
    Iterable<DiskRange> mergedRanges = mergeAdjacentDiskRanges(diskRanges.values(), maxMergeDistance, maxBufferSize);
    ImmutableMap.Builder<K, OrcDataSourceInput> slices = ImmutableMap.builder();
    if (lazyReadSmallRanges) {
        for (DiskRange mergedRange : mergedRanges) {
            LazyBufferLoader mergedRangeLazyLoader = new LazyBufferLoader(mergedRange);
            for (Entry<K, DiskRange> diskRangeEntry : diskRanges.entrySet()) {
                DiskRange diskRange = diskRangeEntry.getValue();
                if (mergedRange.contains(diskRange)) {
                    FixedLengthSliceInput sliceInput = new LazySliceInput(diskRange.getLength(), new LazyMergedSliceLoader(diskRange, mergedRangeLazyLoader));
                    slices.put(diskRangeEntry.getKey(), new OrcDataSourceInput(sliceInput, diskRange.getLength()));
                }
            }
        }
    } else {
        Map<DiskRange, byte[]> buffers = new LinkedHashMap<>();
        for (DiskRange mergedRange : mergedRanges) {
            // read full range in one request
            byte[] buffer = new byte[mergedRange.getLength()];
            readFully(mergedRange.getOffset(), buffer);
            buffers.put(mergedRange, buffer);
        }
        for (Entry<K, DiskRange> entry : diskRanges.entrySet()) {
            slices.put(entry.getKey(), new OrcDataSourceInput(getDiskRangeSlice(entry.getValue(), buffers).getInput(), entry.getValue().getLength()));
        }
    }
    Map<K, OrcDataSourceInput> sliceStreams = slices.build();
    verify(sliceStreams.keySet().equals(diskRanges.keySet()));
    return sliceStreams;
}