Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 26 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class ArrayCombinationsFunction method combinations.

@TypeParameter("T")
@SqlType("array(array(T))")
public static Block combinations(@TypeParameter("T") Type elementType, @SqlType("array(T)") Block array, @SqlType(StandardTypes.INTEGER) long n) {
    int arrayLength = array.getPositionCount();
    int combinationLength = toIntExact(n);
    checkCondition(combinationLength >= 0, INVALID_FUNCTION_ARGUMENT, "combination size must not be negative: %s", combinationLength);
    checkCondition(combinationLength <= MAX_COMBINATION_LENGTH, INVALID_FUNCTION_ARGUMENT, "combination size must not exceed %s: %s", MAX_COMBINATION_LENGTH, combinationLength);
    ArrayType arrayType = new ArrayType(elementType);
    if (combinationLength > arrayLength) {
        return arrayType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 0).build();
    }
    int combinationCount = combinationCount(arrayLength, combinationLength);
    checkCondition(combinationCount * (long) combinationLength <= MAX_RESULT_ELEMENTS, INVALID_FUNCTION_ARGUMENT, "combinations exceed max size");
    int[] ids = new int[combinationCount * combinationLength];
    int idsPosition = 0;
    int[] combination = firstCombination(combinationLength);
    do {
        arraycopy(combination, 0, ids, idsPosition, combinationLength);
        idsPosition += combinationLength;
    } while (nextCombination(combination, arrayLength));
    verify(idsPosition == ids.length, "idsPosition != ids.length, %s and %s respectively", idsPosition, ids.length);
    int[] offsets = new int[combinationCount + 1];
    setAll(offsets, i -> i * combinationLength);
    return ArrayBlock.fromElementBlock(combinationCount, Optional.empty(), offsets, new DictionaryBlock(array, ids));
}

Example 27 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class InputPageProjection method project.

@Override
public Work<List<Block>> project(SqlFunctionProperties properties, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) {
    Block block = requireNonNull(page, "page is null").getBlock(0);
    requireNonNull(selectedPositions, "selectedPositions is null");
    Block result;
    if (selectedPositions.isList()) {
        result = new DictionaryBlock(selectedPositions.getOffset(), selectedPositions.size(), block.getLoadedBlock(), selectedPositions.getPositions(), false, DictionaryId.randomDictionaryId());
    } else if (selectedPositions.getOffset() == 0 && selectedPositions.size() == page.getPositionCount()) {
        result = block.getLoadedBlock();
    } else {
        result = block.getRegion(selectedPositions.getOffset(), selectedPositions.size());
    }
    return new CompletedWork<>(ImmutableList.of(result));
}

Example 28 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class AbstractTestBlock method assertBlockSize.

private void assertBlockSize(Block block) {
    // Asserting on `block` is not very effective because most blocks passed to this method is compact.
    // Therefore, we split the `block` into two and assert again.
    // ------------------Test Whole Block Sizes---------------------------------------------------
    // Assert sizeInBytes for the whole block.
    long expectedBlockSize = copyBlockViaBlockSerde(block).getSizeInBytes();
    assertEquals(block.getSizeInBytes(), expectedBlockSize);
    assertEquals(block.getRegionSizeInBytes(0, block.getPositionCount()), expectedBlockSize);
    // Assert logicalSize for the whole block. Note that copyBlockViaBlockSerde would flatten DictionaryBlock or RleBlock
    long logicalSizeInBytes = block.getLogicalSizeInBytes();
    long expectedLogicalBlockSize = copyBlockViaBlockSerde(block).getLogicalSizeInBytes();
    assertEquals(logicalSizeInBytes, expectedLogicalBlockSize);
    assertEquals(block.getRegionLogicalSizeInBytes(0, block.getPositionCount()), expectedLogicalBlockSize);
    // Assert approximateLogicalSize for the whole block
    long approximateLogicalSizeInBytes = block.getApproximateRegionLogicalSizeInBytes(0, block.getPositionCount());
    long expectedApproximateLogicalBlockSize = expectedLogicalBlockSize;
    if (block instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) block).getDictionary().getPositionCount();
        expectedApproximateLogicalBlockSize = ((DictionaryBlock) block).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * block.getPositionCount() / dictionaryPositionCount;
    }
    assertEquals(approximateLogicalSizeInBytes, expectedApproximateLogicalBlockSize);
    // ------------------Test First Half Sizes---------------------------------------------------
    List<Block> splitBlock = splitBlock(block, 2);
    Block firstHalf = splitBlock.get(0);
    int firstHalfPositionCount = firstHalf.getPositionCount();
    // Assert sizeInBytes for the firstHalf block.
    long expectedFirstHalfSize = copyBlockViaBlockSerde(firstHalf).getSizeInBytes();
    assertEquals(firstHalf.getSizeInBytes(), expectedFirstHalfSize);
    assertEquals(block.getRegionSizeInBytes(0, firstHalfPositionCount), expectedFirstHalfSize);
    // Assert logicalSize for the firstHalf block
    long firstHalfLogicalSizeInBytes = firstHalf.getLogicalSizeInBytes();
    long expectedFirstHalfLogicalSize = copyBlockViaBlockSerde(firstHalf).getLogicalSizeInBytes();
    assertEquals(firstHalfLogicalSizeInBytes, expectedFirstHalfLogicalSize);
    assertEquals(firstHalf.getRegionLogicalSizeInBytes(0, firstHalfPositionCount), expectedFirstHalfLogicalSize);
    // Assert approximateLogicalSize for the firstHalf block using logicalSize
    long approximateFirstHalfLogicalSize = firstHalf.getApproximateRegionLogicalSizeInBytes(0, firstHalfPositionCount);
    long expectedApproximateFirstHalfLogicalSize = expectedFirstHalfLogicalSize;
    if (firstHalf instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) firstHalf).getDictionary().getPositionCount();
        expectedApproximateFirstHalfLogicalSize = ((DictionaryBlock) firstHalf).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * firstHalfPositionCount / dictionaryPositionCount;
    }
    assertEquals(approximateFirstHalfLogicalSize, expectedApproximateFirstHalfLogicalSize);
    // Assert approximateLogicalSize for the firstHalf block using the ratio of firstHalf logicalSize vs whole block logicalSize
    long expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock = logicalSizeInBytes == 0 ? approximateLogicalSizeInBytes : approximateLogicalSizeInBytes * firstHalfLogicalSizeInBytes / logicalSizeInBytes;
    assertBetweenInclusive(approximateFirstHalfLogicalSize, // Allow for some error margins due to skew in blocks
    min(expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock - 3, (long) (expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock * 0.7)), max(expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock + 3, (long) (expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock * 1.3)));
    // ------------------Test Second Half Sizes---------------------------------------------------
    Block secondHalf = splitBlock.get(1);
    int secondHalfPositionCount = secondHalf.getPositionCount();
    // Assert sizeInBytes for the secondHalf block.
    long expectedSecondHalfSize = copyBlockViaBlockSerde(secondHalf).getSizeInBytes();
    assertEquals(secondHalf.getSizeInBytes(), expectedSecondHalfSize);
    assertEquals(block.getRegionSizeInBytes(firstHalfPositionCount, secondHalfPositionCount), expectedSecondHalfSize);
    // Assert logicalSize for the secondHalf block.
    long secondHalfLogicalSizeInBytes = secondHalf.getLogicalSizeInBytes();
    long expectedSecondHalfLogicalSize = copyBlockViaBlockSerde(secondHalf).getLogicalSizeInBytes();
    assertEquals(secondHalfLogicalSizeInBytes, expectedSecondHalfLogicalSize);
    assertEquals(secondHalf.getRegionLogicalSizeInBytes(0, secondHalfPositionCount), expectedSecondHalfLogicalSize);
    // Assert approximateLogicalSize for the secondHalf block using logicalSize
    long approximateSecondHalfLogicalSize = secondHalf.getApproximateRegionLogicalSizeInBytes(0, secondHalfPositionCount);
    long expectedApproximateSecondHalfLogicalSize = copyBlockViaBlockSerde(secondHalf).getApproximateRegionLogicalSizeInBytes(0, secondHalfPositionCount);
    if (secondHalf instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) secondHalf).getDictionary().getPositionCount();
        expectedApproximateSecondHalfLogicalSize = ((DictionaryBlock) secondHalf).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * secondHalfPositionCount / dictionaryPositionCount;
    }
    assertEquals(approximateSecondHalfLogicalSize, expectedApproximateSecondHalfLogicalSize);
    // Assert approximateLogicalSize for the secondHalf block using the ratio of firstHalf logicalSize vs whole block logicalSize
    long expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock = logicalSizeInBytes == 0 ? approximateLogicalSizeInBytes : approximateLogicalSizeInBytes * secondHalfLogicalSizeInBytes / logicalSizeInBytes;
    assertBetweenInclusive(approximateSecondHalfLogicalSize, // Allow for some error margins due to skew in blocks
    min(expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock - 3, (long) (expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock * 0.7)), max(expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock + 3, (long) (expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock * 1.3)));
    // ----------------Test getPositionsSizeInBytes----------------------------------------
    boolean[] positions = new boolean[block.getPositionCount()];
    fill(positions, 0, firstHalfPositionCount, true);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedFirstHalfSize);
    fill(positions, true);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedBlockSize);
    fill(positions, 0, firstHalfPositionCount, false);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedSecondHalfSize);
}

Example 29 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class OptimizedTypedSet method getBlock.

/**
 * Build and return the block representing this set
 */
public Block getBlock() {
    if (size == 0) {
        return elementType.createBlockBuilder(null, 0).build();
    }
    if (currentBlockIndex == 0) {
        // Just one block. Return a DictionaryBlock
        Block block = blocks[currentBlockIndex];
        SelectedPositions selectedPositions = getPositionsForBlocks().get(currentBlockIndex);
        return new DictionaryBlock(selectedPositions.getOffset(), selectedPositions.size(), block, selectedPositions.getPositions(), false, DictionaryId.randomDictionaryId());
    }
    Block firstBlock = blocks[0];
    BlockBuilder blockBuilder = elementType.createBlockBuilder(null, size, toIntExact(firstBlock.getApproximateRegionLogicalSizeInBytes(0, firstBlock.getPositionCount()) / max(1, toIntExact(firstBlock.getPositionCount()))));
    for (int i = 0; i <= currentBlockIndex; i++) {
        Block block = blocks[i];
        SelectedPositions selectedPositions = getPositionsForBlocks().get(i);
        int positionCount = selectedPositions.size();
        if (!selectedPositions.isList()) {
            if (positionCount == block.getPositionCount()) {
                return block;
            } else {
                return block.getRegion(selectedPositions.getOffset(), positionCount);
            }
        }
        int[] positions = selectedPositions.getPositions();
        for (int j = 0; j < positionCount; j++) {
            // offset is always 0
            int position = positions[j];
            if (block.isNull(position)) {
                blockBuilder.appendNull();
            } else {
                elementType.appendTo(block, position, blockBuilder);
            }
        }
    }
    return blockBuilder.build();
}

Example 30 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class BlockAssertions method createLongDictionaryBlock.

public static Block createLongDictionaryBlock(int start, int length) {
    checkArgument(length > 5, "block must have more than 5 entries");
    int dictionarySize = length / 5;
    BlockBuilder builder = BIGINT.createBlockBuilder(null, dictionarySize);
    for (int i = start; i < start + dictionarySize; i++) {
        BIGINT.writeLong(builder, i);
    }
    int[] ids = new int[length];
    for (int i = 0; i < length; i++) {
        ids[i] = i % dictionarySize;
    }
    return new DictionaryBlock(builder.build(), ids);
}