Examples with SliceArrayBlock com.facebook.presto.spi.block.SliceArrayBlock used on opensource projects

Example 6 with SliceArrayBlock

use of com.facebook.presto.spi.block.SliceArrayBlock in project presto by prestodb.

the class SliceDirectStreamReader method readBlock.

@Override
public Block readBlock(Type type) throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the length reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            if (lengthStream == null) {
                throw new OrcCorruptionException("Value is not null but length stream is not present");
            }
            long dataSkipSize = lengthStream.sum(readOffset);
            if (dataSkipSize > 0) {
                if (dataStream == null) {
                    throw new OrcCorruptionException("Value is not null but data stream is not present");
                }
                dataStream.skip(dataSkipSize);
            }
        }
    }
    if (isNullVector.length < nextBatchSize) {
        isNullVector = new boolean[nextBatchSize];
    }
    if (lengthVector.length < nextBatchSize) {
        lengthVector = new int[nextBatchSize];
    }
    if (presentStream == null) {
        if (lengthStream == null) {
            throw new OrcCorruptionException("Value is not null but length stream is not present");
        }
        Arrays.fill(isNullVector, false);
        lengthStream.nextIntVector(nextBatchSize, lengthVector);
    } else {
        int nullValues = presentStream.getUnsetBits(nextBatchSize, isNullVector);
        if (nullValues != nextBatchSize) {
            if (lengthStream == null) {
                throw new OrcCorruptionException("Value is not null but length stream is not present");
            }
            lengthStream.nextIntVector(nextBatchSize, lengthVector, isNullVector);
        }
    }
    int totalLength = 0;
    for (int i = 0; i < nextBatchSize; i++) {
        if (!isNullVector[i]) {
            totalLength += lengthVector[i];
        }
    }
    byte[] data = EMPTY_BYTE_ARRAY;
    if (totalLength > 0) {
        if (dataStream == null) {
            throw new OrcCorruptionException("Value is not null but data stream is not present");
        }
        data = dataStream.next(totalLength);
    }
    Slice[] sliceVector = new Slice[nextBatchSize];
    int offset = 0;
    for (int i = 0; i < nextBatchSize; i++) {
        if (!isNullVector[i]) {
            int length = lengthVector[i];
            Slice value = Slices.wrappedBuffer(data, offset, length);
            if (isVarcharType(type)) {
                value = truncateToLength(value, type);
            }
            if (isCharType(type)) {
                value = trimSpacesAndTruncateToLength(value, type);
            }
            sliceVector[i] = value;
            offset += length;
        }
    }
    readOffset = 0;
    nextBatchSize = 0;
    return new SliceArrayBlock(sliceVector.length, sliceVector);
}

Example 7 with SliceArrayBlock

use of com.facebook.presto.spi.block.SliceArrayBlock in project presto by prestodb.

the class TestPage method testCompactDictionaryBlocks.

@Test
public void testCompactDictionaryBlocks() throws Exception {
    int positionCount = 100;
    // Create 2 dictionary blocks with the same source id
    DictionaryId commonSourceId = randomDictionaryId();
    int commonDictionaryUsedPositions = 20;
    int[] commonDictionaryIds = getDictionaryIds(positionCount, commonDictionaryUsedPositions);
    // first dictionary contains "varbinary" values
    Slice[] dictionaryValues1 = createExpectedValues(50);
    SliceArrayBlock dictionary1 = new SliceArrayBlock(dictionaryValues1.length, dictionaryValues1);
    DictionaryBlock commonSourceIdBlock1 = new DictionaryBlock(positionCount, dictionary1, commonDictionaryIds, commonSourceId);
    // second dictionary block is "length(firstColumn)"
    BlockBuilder dictionary2 = BIGINT.createBlockBuilder(new BlockBuilderStatus(), dictionary1.getPositionCount());
    for (Slice expectedValue : dictionaryValues1) {
        BIGINT.writeLong(dictionary2, expectedValue.length());
    }
    DictionaryBlock commonSourceIdBlock2 = new DictionaryBlock(positionCount, dictionary2.build(), commonDictionaryIds, commonSourceId);
    // Create block with a different source id, dictionary size, used
    int otherDictionaryUsedPositions = 30;
    int[] otherDictionaryIds = getDictionaryIds(positionCount, otherDictionaryUsedPositions);
    SliceArrayBlock dictionary3 = new SliceArrayBlock(70, createExpectedValues(70));
    DictionaryBlock randomSourceIdBlock = new DictionaryBlock(positionCount, dictionary3, otherDictionaryIds);
    Page page = new Page(commonSourceIdBlock1, randomSourceIdBlock, commonSourceIdBlock2);
    page.compact();
    // dictionary blocks should all be compact
    assertTrue(((DictionaryBlock) page.getBlock(0)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(1)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(2)).isCompact());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(1)).getDictionary().getPositionCount(), otherDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(2)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    // Blocks that had the same source id before compacting page should have the same source id after compacting page
    assertNotEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(1)).getDictionarySourceId());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(2)).getDictionarySourceId());
}