Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 16 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestDictionaryBlock method testCompactGetPositions.

@Test
public void testCompactGetPositions() {
    DictionaryBlock block = new DictionaryBlock(createSlicesBlock(createExpectedValues(10)), new int[] { 0, 1, 2, 3, 4, 5 }).compact();
    // 3, 3, 4, 5, 2, 0, 1, 1
    block = (DictionaryBlock) block.getPositions(new int[] { 3, 3, 4, 5, 2, 0, 1, 1 }, 0, 7);
    assertTrue(block.isCompact());
    // 3, 3, 4, 5, 2, 0, 1, 1, 0, 2, 5, 4, 3
    block = (DictionaryBlock) block.getPositions(new int[] { 0, 1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1 }, 0, 12);
    assertTrue(block.isCompact());
    // 3, 4, 3, 4, 3
    block = (DictionaryBlock) block.getPositions(new int[] { 0, 2, 0, 2, 0 }, 0, 5);
    assertFalse(block.isCompact());
    block = block.compact();
    // 3, 4, 4, 4
    block = (DictionaryBlock) block.getPositions(new int[] { 0, 1, 1, 1 }, 0, 4);
    assertTrue(block.isCompact());
    // 4, 4, 4, 4
    block = (DictionaryBlock) block.getPositions(new int[] { 1, 1, 1, 1 }, 0, 4);
    assertFalse(block.isCompact());
    block = block.compact();
    // 4
    block = (DictionaryBlock) block.getPositions(new int[] { 0 }, 0, 1);
    assertTrue(block.isCompact());
    // empty
    block = (DictionaryBlock) block.getPositions(new int[] {}, 0, 0);
    assertFalse(block.isCompact());
    block = block.compact();
    // empty
    block = (DictionaryBlock) block.getPositions(new int[] {}, 0, 0);
    assertTrue(block.isCompact());
}

Example 17 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class AbstractTestBlock method assertBlockSize.

private void assertBlockSize(Block block) {
    // Asserting on `block` is not very effective because most blocks passed to this method is compact.
    // Therefore, we split the `block` into two and assert again.
    // ------------------Test Whole Block Sizes---------------------------------------------------
    // Assert sizeInBytes for the whole block.
    long expectedBlockSize = copyBlockViaBlockSerde(block).getSizeInBytes();
    assertEquals(block.getSizeInBytes(), expectedBlockSize);
    assertEquals(block.getRegionSizeInBytes(0, block.getPositionCount()), expectedBlockSize);
    // Assert logicalSize for the whole block. Note that copyBlockViaBlockSerde would flatten DictionaryBlock or RleBlock
    long logicalSizeInBytes = block.getLogicalSizeInBytes();
    long expectedLogicalBlockSize = copyBlockViaBlockSerde(block).getLogicalSizeInBytes();
    assertEquals(logicalSizeInBytes, expectedLogicalBlockSize);
    assertEquals(block.getRegionLogicalSizeInBytes(0, block.getPositionCount()), expectedLogicalBlockSize);
    // Assert approximateLogicalSize for the whole block
    long approximateLogicalSizeInBytes = block.getApproximateRegionLogicalSizeInBytes(0, block.getPositionCount());
    long expectedApproximateLogicalBlockSize = expectedLogicalBlockSize;
    if (block instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) block).getDictionary().getPositionCount();
        expectedApproximateLogicalBlockSize = ((DictionaryBlock) block).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * block.getPositionCount() / dictionaryPositionCount;
    }
    assertEquals(approximateLogicalSizeInBytes, expectedApproximateLogicalBlockSize);
    // ------------------Test First Half Sizes---------------------------------------------------
    List<Block> splitBlock = splitBlock(block, 2);
    Block firstHalf = splitBlock.get(0);
    int firstHalfPositionCount = firstHalf.getPositionCount();
    // Assert sizeInBytes for the firstHalf block.
    long expectedFirstHalfSize = copyBlockViaBlockSerde(firstHalf).getSizeInBytes();
    assertEquals(firstHalf.getSizeInBytes(), expectedFirstHalfSize);
    assertEquals(block.getRegionSizeInBytes(0, firstHalfPositionCount), expectedFirstHalfSize);
    // Assert logicalSize for the firstHalf block
    long firstHalfLogicalSizeInBytes = firstHalf.getLogicalSizeInBytes();
    long expectedFirstHalfLogicalSize = copyBlockViaBlockSerde(firstHalf).getLogicalSizeInBytes();
    assertEquals(firstHalfLogicalSizeInBytes, expectedFirstHalfLogicalSize);
    assertEquals(firstHalf.getRegionLogicalSizeInBytes(0, firstHalfPositionCount), expectedFirstHalfLogicalSize);
    // Assert approximateLogicalSize for the firstHalf block using logicalSize
    long approximateFirstHalfLogicalSize = firstHalf.getApproximateRegionLogicalSizeInBytes(0, firstHalfPositionCount);
    long expectedApproximateFirstHalfLogicalSize = expectedFirstHalfLogicalSize;
    if (firstHalf instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) firstHalf).getDictionary().getPositionCount();
        expectedApproximateFirstHalfLogicalSize = ((DictionaryBlock) firstHalf).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * firstHalfPositionCount / dictionaryPositionCount;
    }
    assertEquals(approximateFirstHalfLogicalSize, expectedApproximateFirstHalfLogicalSize);
    // Assert approximateLogicalSize for the firstHalf block using the ratio of firstHalf logicalSize vs whole block logicalSize
    long expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock = logicalSizeInBytes == 0 ? approximateLogicalSizeInBytes : approximateLogicalSizeInBytes * firstHalfLogicalSizeInBytes / logicalSizeInBytes;
    assertBetweenInclusive(approximateFirstHalfLogicalSize, // Allow for some error margins due to skew in blocks
    min(expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock - 3, (long) (expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock * 0.7)), max(expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock + 3, (long) (expectedApproximateFirstHalfLogicalSizeFromUnsplittedBlock * 1.3)));
    // ------------------Test Second Half Sizes---------------------------------------------------
    Block secondHalf = splitBlock.get(1);
    int secondHalfPositionCount = secondHalf.getPositionCount();
    // Assert sizeInBytes for the secondHalf block.
    long expectedSecondHalfSize = copyBlockViaBlockSerde(secondHalf).getSizeInBytes();
    assertEquals(secondHalf.getSizeInBytes(), expectedSecondHalfSize);
    assertEquals(block.getRegionSizeInBytes(firstHalfPositionCount, secondHalfPositionCount), expectedSecondHalfSize);
    // Assert logicalSize for the secondHalf block.
    long secondHalfLogicalSizeInBytes = secondHalf.getLogicalSizeInBytes();
    long expectedSecondHalfLogicalSize = copyBlockViaBlockSerde(secondHalf).getLogicalSizeInBytes();
    assertEquals(secondHalfLogicalSizeInBytes, expectedSecondHalfLogicalSize);
    assertEquals(secondHalf.getRegionLogicalSizeInBytes(0, secondHalfPositionCount), expectedSecondHalfLogicalSize);
    // Assert approximateLogicalSize for the secondHalf block using logicalSize
    long approximateSecondHalfLogicalSize = secondHalf.getApproximateRegionLogicalSizeInBytes(0, secondHalfPositionCount);
    long expectedApproximateSecondHalfLogicalSize = copyBlockViaBlockSerde(secondHalf).getApproximateRegionLogicalSizeInBytes(0, secondHalfPositionCount);
    if (secondHalf instanceof DictionaryBlock) {
        int dictionaryPositionCount = ((DictionaryBlock) secondHalf).getDictionary().getPositionCount();
        expectedApproximateSecondHalfLogicalSize = ((DictionaryBlock) secondHalf).getDictionary().getApproximateRegionLogicalSizeInBytes(0, dictionaryPositionCount) * secondHalfPositionCount / dictionaryPositionCount;
    }
    assertEquals(approximateSecondHalfLogicalSize, expectedApproximateSecondHalfLogicalSize);
    // Assert approximateLogicalSize for the secondHalf block using the ratio of firstHalf logicalSize vs whole block logicalSize
    long expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock = logicalSizeInBytes == 0 ? approximateLogicalSizeInBytes : approximateLogicalSizeInBytes * secondHalfLogicalSizeInBytes / logicalSizeInBytes;
    assertBetweenInclusive(approximateSecondHalfLogicalSize, // Allow for some error margins due to skew in blocks
    min(expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock - 3, (long) (expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock * 0.7)), max(expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock + 3, (long) (expectedApproximateSecondHalfLogicalSizeFromUnsplittedBlock * 1.3)));
    // ----------------Test getPositionsSizeInBytes----------------------------------------
    boolean[] positions = new boolean[block.getPositionCount()];
    fill(positions, 0, firstHalfPositionCount, true);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedFirstHalfSize);
    fill(positions, true);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedBlockSize);
    fill(positions, 0, firstHalfPositionCount, false);
    assertEquals(block.getPositionsSizeInBytes(positions), expectedSecondHalfSize);
}

Example 18 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestBlockEncodingBuffers method buildDictionaryBlockStatus.

private BlockStatus buildDictionaryBlockStatus(BlockStatus dictionary, int positionCount) {
    DictionaryBlock dictionaryBlock = createRandomDictionaryBlock(dictionary.block, positionCount);
    int[] mappedExpectedRowSizes = IntStream.range(0, positionCount).map(i -> dictionary.expectedRowSizes[dictionaryBlock.getId(i)]).toArray();
    return new BlockStatus(dictionaryBlock, mappedExpectedRowSizes);
}

Example 19 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestDictionaryAwarePageProjection method createDictionaryBlockWithFailure.

private static DictionaryBlock createDictionaryBlockWithFailure(int dictionarySize, int blockSize) {
    Block dictionary = createLongSequenceBlock(-10, dictionarySize - 10);
    int[] ids = new int[blockSize];
    Arrays.setAll(ids, index -> index % dictionarySize);
    return new DictionaryBlock(dictionary, ids);
}

Example 20 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestDictionaryAwarePageProjection method testDictionaryBlock.

@Test(dataProvider = "forceYield")
public void testDictionaryBlock(boolean forceYield) {
    DictionaryBlock block = createDictionaryBlock(10, 100);
    testProject(block, DictionaryBlock.class, forceYield);
}