Search in sources :

Example 1 with BooleanArrayList

use of it.unimi.dsi.fastutil.booleans.BooleanArrayList in project presto by prestodb.

the class ParquetReader method readMap.

private ColumnChunk readMap(GroupField field) throws IOException {
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size());
    Block[] blocks = new Block[parameters.size()];
    ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get());
    blocks[0] = columnChunk.getBlock();
    blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock();
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();
    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(offsets.size() - 1, Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]);
    return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
Also used : BooleanList(it.unimi.dsi.fastutil.booleans.BooleanList) MapType(com.facebook.presto.common.type.MapType) Type(com.facebook.presto.common.type.Type) BooleanArrayList(it.unimi.dsi.fastutil.booleans.BooleanArrayList) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) IntArrayBlock(com.facebook.presto.common.block.IntArrayBlock) RowBlock(com.facebook.presto.common.block.RowBlock) ArrayBlock(com.facebook.presto.common.block.ArrayBlock) LongArrayBlock(com.facebook.presto.common.block.LongArrayBlock) Block(com.facebook.presto.common.block.Block) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) MapType(com.facebook.presto.common.type.MapType) IntList(it.unimi.dsi.fastutil.ints.IntList)

Example 2 with BooleanArrayList

use of it.unimi.dsi.fastutil.booleans.BooleanArrayList in project presto by prestodb.

the class ParquetReader method readArray.

private ColumnChunk readArray(GroupField field) throws IOException {
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size());
    Field elementField = field.getChildren().get(0).get();
    ColumnChunk columnChunk = readColumnChunk(elementField);
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();
    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock());
    return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
Also used : BooleanList(it.unimi.dsi.fastutil.booleans.BooleanList) GroupField(com.facebook.presto.parquet.GroupField) PrimitiveField(com.facebook.presto.parquet.PrimitiveField) Field(com.facebook.presto.parquet.Field) MapType(com.facebook.presto.common.type.MapType) Type(com.facebook.presto.common.type.Type) BooleanArrayList(it.unimi.dsi.fastutil.booleans.BooleanArrayList) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) IntArrayBlock(com.facebook.presto.common.block.IntArrayBlock) RowBlock(com.facebook.presto.common.block.RowBlock) ArrayBlock(com.facebook.presto.common.block.ArrayBlock) LongArrayBlock(com.facebook.presto.common.block.LongArrayBlock) Block(com.facebook.presto.common.block.Block) IntArrayList(it.unimi.dsi.fastutil.ints.IntArrayList) IntList(it.unimi.dsi.fastutil.ints.IntList)

Example 3 with BooleanArrayList

use of it.unimi.dsi.fastutil.booleans.BooleanArrayList in project presto by prestodb.

the class StructColumnReader method calculateStructOffsets.

/**
 * Each struct has three variants of presence:
 * 1) Struct is not defined, because one of it's optional parent fields is null
 * 2) Struct is null
 * 3) Struct is defined and not empty.
 */
public static BooleanList calculateStructOffsets(Field field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) {
    int maxDefinitionLevel = field.getDefinitionLevel();
    int maxRepetitionLevel = field.getRepetitionLevel();
    BooleanList structIsNull = new BooleanArrayList();
    boolean required = field.isRequired();
    if (fieldDefinitionLevels == null) {
        return structIsNull;
    }
    for (int i = 0; i < fieldDefinitionLevels.length; i++) {
        if (fieldRepetitionLevels[i] <= maxRepetitionLevel) {
            if (isValueNull(required, fieldDefinitionLevels[i], maxDefinitionLevel)) {
                // Struct is null
                structIsNull.add(true);
            } else if (fieldDefinitionLevels[i] >= maxDefinitionLevel) {
                // Struct is defined and not empty
                structIsNull.add(false);
            }
        }
    }
    return structIsNull;
}
Also used : BooleanList(it.unimi.dsi.fastutil.booleans.BooleanList) BooleanArrayList(it.unimi.dsi.fastutil.booleans.BooleanArrayList)

Example 4 with BooleanArrayList

use of it.unimi.dsi.fastutil.booleans.BooleanArrayList in project presto by prestodb.

the class TestBooleanStream method testWriteMultiple.

@Test
public void testWriteMultiple() throws IOException {
    BooleanOutputStream outputStream = createValueOutputStream();
    for (int i = 0; i < 3; i++) {
        outputStream.reset();
        BooleanList expectedValues = new BooleanArrayList(1024);
        outputStream.writeBooleans(32, true);
        expectedValues.addAll(Collections.nCopies(32, true));
        outputStream.writeBooleans(32, false);
        expectedValues.addAll(Collections.nCopies(32, false));
        outputStream.writeBooleans(1, true);
        expectedValues.add(true);
        outputStream.writeBooleans(1, false);
        expectedValues.add(false);
        outputStream.writeBooleans(34, true);
        expectedValues.addAll(Collections.nCopies(34, true));
        outputStream.writeBooleans(34, false);
        expectedValues.addAll(Collections.nCopies(34, false));
        outputStream.writeBoolean(true);
        expectedValues.add(true);
        outputStream.writeBoolean(false);
        expectedValues.add(false);
        outputStream.close();
        BooleanInputStream valueStream = createValueStream(outputStream);
        for (int index = 0; index < expectedValues.size(); index++) {
            boolean expectedValue = expectedValues.getBoolean(index);
            boolean actualValue = readValue(valueStream);
            assertEquals(actualValue, expectedValue);
        }
    }
}
Also used : BooleanList(it.unimi.dsi.fastutil.booleans.BooleanList) BooleanArrayList(it.unimi.dsi.fastutil.booleans.BooleanArrayList) BooleanStreamCheckpoint(com.facebook.presto.orc.checkpoint.BooleanStreamCheckpoint) Test(org.testng.annotations.Test)

Aggregations

BooleanArrayList (it.unimi.dsi.fastutil.booleans.BooleanArrayList)4 BooleanList (it.unimi.dsi.fastutil.booleans.BooleanList)4 ArrayBlock (com.facebook.presto.common.block.ArrayBlock)2 Block (com.facebook.presto.common.block.Block)2 IntArrayBlock (com.facebook.presto.common.block.IntArrayBlock)2 LongArrayBlock (com.facebook.presto.common.block.LongArrayBlock)2 RowBlock (com.facebook.presto.common.block.RowBlock)2 RunLengthEncodedBlock (com.facebook.presto.common.block.RunLengthEncodedBlock)2 MapType (com.facebook.presto.common.type.MapType)2 Type (com.facebook.presto.common.type.Type)2 IntArrayList (it.unimi.dsi.fastutil.ints.IntArrayList)2 IntList (it.unimi.dsi.fastutil.ints.IntList)2 BooleanStreamCheckpoint (com.facebook.presto.orc.checkpoint.BooleanStreamCheckpoint)1 Field (com.facebook.presto.parquet.Field)1 GroupField (com.facebook.presto.parquet.GroupField)1 PrimitiveField (com.facebook.presto.parquet.PrimitiveField)1 Test (org.testng.annotations.Test)1