Examples with VariableWidthBlock com.facebook.presto.common.block.VariableWidthBlock used on opensource projects

Example 6 with VariableWidthBlock

use of com.facebook.presto.common.block.VariableWidthBlock in project presto by prestodb.

the class TestVariableWidthBlock method testCompactBlock.

@Test
public void testCompactBlock() {
    Slice compactSlice = Slices.copyOf(createExpectedValue(16));
    Slice incompactSlice = Slices.copyOf(createExpectedValue(20)).slice(0, 16);
    int[] offsets = { 0, 1, 1, 2, 4, 8, 16 };
    boolean[] valueIsNull = { false, true, false, false, false, false };
    testCompactBlock(new VariableWidthBlock(0, EMPTY_SLICE, new int[1], Optional.empty()));
    testCompactBlock(new VariableWidthBlock(valueIsNull.length, compactSlice, offsets, Optional.of(valueIsNull)));
    testIncompactBlock(new VariableWidthBlock(valueIsNull.length - 1, compactSlice, offsets, Optional.of(valueIsNull)));
    // underlying slice is not compact
    testIncompactBlock(new VariableWidthBlock(valueIsNull.length, incompactSlice, offsets, Optional.of(valueIsNull)));
}

Example 7 with VariableWidthBlock

use of com.facebook.presto.common.block.VariableWidthBlock in project presto by prestodb.

the class BinaryNestedBatchReader method readNestedWithNull.

@Override
protected ColumnChunk readNestedWithNull() throws IOException {
    int maxDefinitionLevel = columnDescriptor.getMaxDefinitionLevel();
    RepetitionLevelDecodingContext repetitionLevelDecodingContext = readRepetitionLevels(nextBatchSize);
    DefinitionLevelDecodingContext definitionLevelDecodingContext = readDefinitionLevels(repetitionLevelDecodingContext.getDLValuesDecoderContexts(), repetitionLevelDecodingContext.getRepetitionLevels().length);
    int[] definitionLevels = definitionLevelDecodingContext.getDefinitionLevels();
    int newBatchSize = 0;
    int batchNonNullCount = 0;
    for (ValuesDecoderContext valuesDecoderContext : definitionLevelDecodingContext.getValuesDecoderContexts()) {
        int nonNullCount = 0;
        int valueCount = 0;
        for (int i = valuesDecoderContext.getStart(); i < valuesDecoderContext.getEnd(); i++) {
            nonNullCount += (definitionLevels[i] == maxDefinitionLevel ? 1 : 0);
            valueCount += (definitionLevels[i] >= maxDefinitionLevel - 1 ? 1 : 0);
        }
        batchNonNullCount += nonNullCount;
        newBatchSize += valueCount;
        valuesDecoderContext.setNonNullCount(nonNullCount);
        valuesDecoderContext.setValueCount(valueCount);
    }
    if (batchNonNullCount == 0) {
        Block block = RunLengthEncodedBlock.create(field.getType(), null, newBatchSize);
        return new ColumnChunk(block, definitionLevels, repetitionLevelDecodingContext.getRepetitionLevels());
    }
    List<ValueBuffer> valueBuffers = new ArrayList<>();
    int bufferSize = 0;
    for (ValuesDecoderContext valuesDecoderContext : definitionLevelDecodingContext.getValuesDecoderContexts()) {
        ValueBuffer valueBuffer = ((BinaryValuesDecoder) valuesDecoderContext.getValuesDecoder()).readNext(valuesDecoderContext.getNonNullCount());
        bufferSize += valueBuffer.getBufferSize();
        valueBuffers.add(valueBuffer);
    }
    byte[] byteBuffer = new byte[bufferSize];
    int[] offsets = new int[newBatchSize + 1];
    int i = 0;
    int bufferIndex = 0;
    int offsetIndex = 0;
    for (ValuesDecoderContext valuesDecoderContext : definitionLevelDecodingContext.getValuesDecoderContexts()) {
        ValueBuffer value = valueBuffers.get(i);
        bufferIndex = ((BinaryValuesDecoder) valuesDecoderContext.getValuesDecoder()).readIntoBuffer(byteBuffer, bufferIndex, offsets, offsetIndex, value);
        offsetIndex += valuesDecoderContext.getValueCount();
        i++;
    }
    boolean[] isNull = new boolean[newBatchSize];
    int offset = 0;
    for (ValuesDecoderContext valuesDecoderContext : definitionLevelDecodingContext.getValuesDecoderContexts()) {
        int destinationIndex = offset + valuesDecoderContext.getValueCount() - 1;
        int sourceIndex = offset + valuesDecoderContext.getNonNullCount() - 1;
        int definitionLevelIndex = valuesDecoderContext.getEnd() - 1;
        offsets[destinationIndex + 1] = offsets[sourceIndex + 1];
        while (destinationIndex >= offset) {
            if (definitionLevels[definitionLevelIndex] == maxDefinitionLevel) {
                offsets[destinationIndex--] = offsets[sourceIndex--];
            } else if (definitionLevels[definitionLevelIndex] == maxDefinitionLevel - 1) {
                offsets[destinationIndex] = offsets[sourceIndex + 1];
                isNull[destinationIndex] = true;
                destinationIndex--;
            }
            definitionLevelIndex--;
        }
        offset += valuesDecoderContext.getValueCount();
    }
    Slice buffer = Slices.wrappedBuffer(byteBuffer, 0, bufferSize);
    boolean hasNoNull = batchNonNullCount == newBatchSize;
    Block block = new VariableWidthBlock(newBatchSize, buffer, offsets, hasNoNull ? Optional.empty() : Optional.of(isNull));
    return new ColumnChunk(block, definitionLevels, repetitionLevelDecodingContext.getRepetitionLevels());
}

Example 8 with VariableWidthBlock

use of com.facebook.presto.common.block.VariableWidthBlock in project presto by prestodb.

the class BinaryFlatBatchReader method readWithNull.

private ColumnChunk readWithNull() throws IOException {
    boolean[] isNull = new boolean[nextBatchSize];
    List<ValueBuffer> valueBuffers = new ArrayList<>();
    List<ValuesDecoderContext> valuesDecoderContexts = new ArrayList<>();
    int bufferSize = 0;
    int totalNonNullCount = 0;
    int remainingInBatch = nextBatchSize;
    int startOffset = 0;
    while (remainingInBatch > 0) {
        if (remainingCountInPage == 0) {
            if (!readNextPage()) {
                break;
            }
        }
        int readChunkSize = Math.min(remainingCountInPage, remainingInBatch);
        int nonNullCount = definitionLevelDecoder.readNext(isNull, startOffset, readChunkSize);
        totalNonNullCount += nonNullCount;
        ValueBuffer valueBuffer = valuesDecoder.readNext(nonNullCount);
        bufferSize += valueBuffer.getBufferSize();
        valueBuffers.add(valueBuffer);
        ValuesDecoderContext<BinaryValuesDecoder> valuesDecoderContext = new ValuesDecoderContext(valuesDecoder, startOffset, startOffset + readChunkSize);
        valuesDecoderContext.setValueCount(readChunkSize);
        valuesDecoderContext.setNonNullCount(nonNullCount);
        valuesDecoderContexts.add(valuesDecoderContext);
        startOffset += readChunkSize;
        remainingInBatch -= readChunkSize;
        remainingCountInPage -= readChunkSize;
    }
    if (totalNonNullCount == 0) {
        Block block = RunLengthEncodedBlock.create(field.getType(), null, nextBatchSize);
        return new ColumnChunk(block, new int[0], new int[0]);
    }
    byte[] byteBuffer = new byte[bufferSize];
    int[] offsets = new int[nextBatchSize + 1];
    int i = 0;
    int bufferIndex = 0;
    int offsetIndex = 0;
    for (ValuesDecoderContext<BinaryValuesDecoder> valuesDecoderContext : valuesDecoderContexts) {
        BinaryValuesDecoder binaryValuesDecoder = valuesDecoderContext.getValuesDecoder();
        ValueBuffer value = valueBuffers.get(i);
        bufferIndex = binaryValuesDecoder.readIntoBuffer(byteBuffer, bufferIndex, offsets, offsetIndex, value);
        offsetIndex += valuesDecoderContext.getValueCount();
        i++;
    }
    Collections.reverse(valuesDecoderContexts);
    for (ValuesDecoderContext valuesDecoderContext : valuesDecoderContexts) {
        int destinationIndex = valuesDecoderContext.getEnd() - 1;
        int sourceIndex = valuesDecoderContext.getStart() + valuesDecoderContext.getNonNullCount() - 1;
        offsets[destinationIndex + 1] = offsets[sourceIndex + 1];
        while (destinationIndex >= valuesDecoderContext.getStart()) {
            if (isNull[destinationIndex]) {
                offsets[destinationIndex] = offsets[sourceIndex + 1];
            } else {
                offsets[destinationIndex] = offsets[sourceIndex];
                sourceIndex--;
            }
            destinationIndex--;
        }
    }
    Slice buffer = Slices.wrappedBuffer(byteBuffer, 0, bufferSize);
    boolean hasNoNull = totalNonNullCount == nextBatchSize;
    Block block = new VariableWidthBlock(nextBatchSize, buffer, offsets, hasNoNull ? Optional.empty() : Optional.of(isNull));
    return new ColumnChunk(block, new int[0], new int[0]);
}

Example 9 with VariableWidthBlock

use of com.facebook.presto.common.block.VariableWidthBlock in project presto by prestodb.

the class SliceDictionarySelectiveReader method wrapDictionaryIfNecessary.

private void wrapDictionaryIfNecessary() {
    if (dictionaryWrapped) {
        return;
    }
    boolean[] isNullVector = new boolean[currentDictionarySize];
    isNullVector[currentDictionarySize - 1] = true;
    byte[] dictionaryDataCopy = Arrays.copyOf(dictionaryData, dictionaryOffsetVector[currentDictionarySize]);
    int[] dictionaryOffsetVectorCopy = Arrays.copyOf(dictionaryOffsetVector, currentDictionarySize + 1);
    dictionary = new VariableWidthBlock(currentDictionarySize, wrappedBuffer(dictionaryDataCopy), dictionaryOffsetVectorCopy, Optional.of(isNullVector));
    dictionaryWrapped = true;
}

Example 10 with VariableWidthBlock

use of com.facebook.presto.common.block.VariableWidthBlock in project presto by prestodb.

the class SliceDirectSelectiveStreamReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount));
    }
    boolean includeNulls = nullsAllowed && presentStream != null;
    if (positionCount != outputPositionCount) {
        compactValues(positions, positionCount, includeNulls);
    }
    return newLease(new VariableWidthBlock(positionCount, dataAsSlice, offsets, Optional.ofNullable(includeNulls ? nulls : null)));
}