Examples with InterleavedBlock com.facebook.presto.spi.block.InterleavedBlock used on opensource projects

Example 1 with InterleavedBlock

use of com.facebook.presto.spi.block.InterleavedBlock in project presto by prestodb.

the class ParquetReader method readMap.

private Block readMap(Type type, List<String> path, IntList elementOffsets) throws IOException {
    List<Type> parameters = type.getTypeParameters();
    checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size());
    Block[] blocks = new Block[parameters.size()];
    IntList keyOffsets = new IntArrayList();
    IntList valueOffsets = new IntArrayList();
    path.add(MAP_TYPE_NAME);
    blocks[0] = readBlock(MAP_KEY_NAME, parameters.get(0), path, keyOffsets);
    blocks[1] = readBlock(MAP_VALUE_NAME, parameters.get(1), path, valueOffsets);
    path.remove(MAP_TYPE_NAME);
    if (blocks[0].getPositionCount() == 0) {
        for (int i = 0; i < batchSize; i++) {
            elementOffsets.add(0);
        }
        return RunLengthEncodedBlock.create(parameters.get(0), null, batchSize);
    }
    InterleavedBlock interleavedBlock = new InterleavedBlock(new Block[] { blocks[0], blocks[1] });
    int[] offsets = new int[batchSize + 1];
    for (int i = 1; i < offsets.length; i++) {
        int elementPositionCount = keyOffsets.getInt(i - 1) * 2;
        elementOffsets.add(elementPositionCount);
        offsets[i] = offsets[i - 1] + elementPositionCount;
    }
    return new ArrayBlock(batchSize, new boolean[batchSize], offsets, interleavedBlock);
}

Example 2 with InterleavedBlock

use of com.facebook.presto.spi.block.InterleavedBlock in project presto by prestodb.

the class MapStreamReader method createKeyValueBlock.

private static InterleavedBlock createKeyValueBlock(int positionCount, Block keys, Block values, int[] lengths) {
    if (!hasNull(keys)) {
        return new InterleavedBlock(new Block[] { keys, values });
    }
    //
    // Map entries with a null key are skipped in the Hive ORC reader, so skip them here also
    //
    IntArrayList nonNullPositions = new IntArrayList(keys.getPositionCount());
    int position = 0;
    for (int mapIndex = 0; mapIndex < positionCount; mapIndex++) {
        int length = lengths[mapIndex];
        for (int entryIndex = 0; entryIndex < length; entryIndex++) {
            if (keys.isNull(position)) {
                // key is null, so remove this entry from the map
                lengths[mapIndex]--;
            } else {
                nonNullPositions.add(position);
            }
            position++;
        }
    }
    Block newKeys = keys.copyPositions(nonNullPositions);
    Block newValues = values.copyPositions(nonNullPositions);
    return new InterleavedBlock(new Block[] { newKeys, newValues });
}

Example 3 with InterleavedBlock

use of com.facebook.presto.spi.block.InterleavedBlock in project presto by prestodb.

the class StructStreamReader method readBlock.

@Override
public Block readBlock(Type type) throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the field readers
            readOffset = presentStream.countBitsSet(readOffset);
        }
        for (StreamReader structField : structFields) {
            structField.prepareNextRead(readOffset);
        }
    }
    List<Type> typeParameters = type.getTypeParameters();
    boolean[] nullVector = new boolean[nextBatchSize];
    Block[] blocks = new Block[typeParameters.size()];
    if (presentStream == null) {
        for (int i = 0; i < typeParameters.size(); i++) {
            StreamReader structField = structFields[i];
            structField.prepareNextRead(nextBatchSize);
            blocks[i] = structField.readBlock(typeParameters.get(i));
        }
    } else {
        int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
        if (nullValues != nextBatchSize) {
            for (int i = 0; i < typeParameters.size(); i++) {
                StreamReader structField = structFields[i];
                structField.prepareNextRead(nextBatchSize - nullValues);
                blocks[i] = structField.readBlock(typeParameters.get(i));
            }
        } else {
            for (int i = 0; i < typeParameters.size(); i++) {
                blocks[i] = typeParameters.get(i).createBlockBuilder(new BlockBuilderStatus(), 0).build();
            }
        }
    }
    // Build offsets for array block (null valued have no positions)
    int[] offsets = new int[nextBatchSize + 1];
    for (int i = 1; i < offsets.length; i++) {
        int length = nullVector[i - 1] ? 0 : typeParameters.size();
        offsets[i] = offsets[i - 1] + length;
    }
    // Struct is represented as an array block holding an interleaved block
    InterleavedBlock interleavedBlock = new InterleavedBlock(blocks);
    ArrayBlock arrayBlock = new ArrayBlock(nextBatchSize, nullVector, offsets, interleavedBlock);
    readOffset = 0;
    nextBatchSize = 0;
    return arrayBlock;
}

Example 4 with InterleavedBlock

use of com.facebook.presto.spi.block.InterleavedBlock in project presto by prestodb.

the class MapStreamReader method readBlock.

@Override
public Block readBlock(Type type) throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the data reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            if (lengthStream == null) {
                throw new OrcCorruptionException("Value is not null but data stream is not present");
            }
            long entrySkipSize = lengthStream.sum(readOffset);
            keyStreamReader.prepareNextRead(toIntExact(entrySkipSize));
            valueStreamReader.prepareNextRead(toIntExact(entrySkipSize));
        }
    }
    // The length vector could be reused, but this simplifies the code below by
    // taking advantage of null entries being initialized to zero.  The vector
    // could be reinitialized for each loop, but that is likely just as expensive
    // as allocating a new array
    int[] lengthVector = new int[nextBatchSize];
    boolean[] nullVector = new boolean[nextBatchSize];
    if (presentStream == null) {
        if (lengthStream == null) {
            throw new OrcCorruptionException("Value is not null but data stream is not present");
        }
        lengthStream.nextIntVector(nextBatchSize, lengthVector);
    } else {
        int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
        if (nullValues != nextBatchSize) {
            if (lengthStream == null) {
                throw new OrcCorruptionException("Value is not null but data stream is not present");
            }
            lengthStream.nextIntVector(nextBatchSize, lengthVector, nullVector);
        }
    }
    Type keyType = type.getTypeParameters().get(0);
    Type valueType = type.getTypeParameters().get(1);
    int entryCount = 0;
    for (int length : lengthVector) {
        entryCount += length;
    }
    Block keys;
    Block values;
    if (entryCount > 0) {
        keyStreamReader.prepareNextRead(entryCount);
        valueStreamReader.prepareNextRead(entryCount);
        keys = keyStreamReader.readBlock(keyType);
        values = valueStreamReader.readBlock(valueType);
    } else {
        keys = keyType.createBlockBuilder(new BlockBuilderStatus(), 0).build();
        values = valueType.createBlockBuilder(new BlockBuilderStatus(), 1).build();
    }
    InterleavedBlock keyValueBlock = createKeyValueBlock(nextBatchSize, keys, values, lengthVector);
    // convert lengths into offsets into the keyValueBlock (e.g., two positions per entry)
    int[] offsets = new int[nextBatchSize + 1];
    for (int i = 1; i < offsets.length; i++) {
        int length = lengthVector[i - 1] * 2;
        offsets[i] = offsets[i - 1] + length;
    }
    ArrayBlock arrayBlock = new ArrayBlock(nextBatchSize, nullVector, offsets, keyValueBlock);
    readOffset = 0;
    nextBatchSize = 0;
    return arrayBlock;
}

Example 5 with InterleavedBlock

use of com.facebook.presto.spi.block.InterleavedBlock in project presto by prestodb.

the class MapConcatFunction method mapConcat.

@UsedByGeneratedCode
public static Block mapConcat(Type keyType, Type valueType, Object state, Block[] maps) {
    int entries = 0;
    int lastMapIndex = maps.length - 1;
    int firstMapIndex = lastMapIndex;
    for (int i = 0; i < maps.length; i++) {
        entries += maps[i].getPositionCount();
        if (maps[i].getPositionCount() > 0) {
            lastMapIndex = i;
            firstMapIndex = min(firstMapIndex, i);
        }
    }
    if (lastMapIndex == firstMapIndex) {
        return maps[lastMapIndex];
    }
    PageBuilder pageBuilder = (PageBuilder) state;
    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }
    // TODO: we should move TypedSet into user state as well
    TypedSet typedSet = new TypedSet(keyType, entries / 2);
    BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(0);
    BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(1);
    // the last map
    Block map = maps[lastMapIndex];
    int total = 0;
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        typedSet.add(map, i);
        keyType.appendTo(map, i, keyBlockBuilder);
        valueType.appendTo(map, i + 1, valueBlockBuilder);
        total++;
    }
    // the map between the last and the first
    for (int idx = lastMapIndex - 1; idx > firstMapIndex; idx--) {
        map = maps[idx];
        for (int i = 0; i < map.getPositionCount(); i += 2) {
            if (!typedSet.contains(map, i)) {
                typedSet.add(map, i);
                keyType.appendTo(map, i, keyBlockBuilder);
                valueType.appendTo(map, i + 1, valueBlockBuilder);
                total++;
            }
        }
    }
    // the first map
    map = maps[firstMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        if (!typedSet.contains(map, i)) {
            keyType.appendTo(map, i, keyBlockBuilder);
            valueType.appendTo(map, i + 1, valueBlockBuilder);
            total++;
        }
    }
    pageBuilder.declarePositions(total);
    Block[] blocks = new Block[2];
    blocks[0] = keyBlockBuilder.getRegion(keyBlockBuilder.getPositionCount() - total, total);
    blocks[1] = valueBlockBuilder.getRegion(valueBlockBuilder.getPositionCount() - total, total);
    return new InterleavedBlock(blocks);
}