Examples with DictionaryBlock io.prestosql.spi.block.DictionaryBlock used on opensource projects

Example 56 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testCopyPositionsWithCompactionsAndReorder.

@Test
public void testCopyPositionsWithCompactionsAndReorder() {
    Slice[] expectedValues = createExpectedValues(10);
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    int[] positionsToCopy = new int[] { 50, 55, 40, 45, 60 };
    DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length);
    assertEquals(copiedBlock.getDictionary().getPositionCount(), 2);
    assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length);
    assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[0], expectedValues[5] });
    assertDictionaryIds(copiedBlock, 0, 1, 0, 1, 0);
}

Example 57 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testCopyPositionsWithCompaction.

@Test
public void testCopyPositionsWithCompaction() {
    Slice[] expectedValues = createExpectedValues(10);
    Slice firstExpectedValue = expectedValues[0];
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    int[] positionsToCopy = new int[] { 0, 10, 20, 30, 40 };
    DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length);
    assertEquals(copiedBlock.getDictionary().getPositionCount(), 1);
    assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length);
    assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] { firstExpectedValue });
    assertBlock(copiedBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue });
}

Example 58 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testSizeInBytes.

@Test
public void testSizeInBytes() {
    Slice[] expectedValues = createExpectedValues(10);
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    assertEquals(dictionaryBlock.getSizeInBytes(), dictionaryBlock.getDictionary().getSizeInBytes() + (100 * SIZE_OF_INT));
}

Example 59 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testLogicalSizeInBytes.

@Test
public void testLogicalSizeInBytes() {
    // The 10 Slices in the array will be of lengths 0 to 9.
    Slice[] expectedValues = createExpectedValues(10);
    // The dictionary within the dictionary block is expected to be a VariableWidthBlock of size 95 bytes.
    // 45 bytes for the expectedValues Slices (sum of seq(0,9)) and 50 bytes for the position and isNull array (total 10 positions).
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    assertEquals(dictionaryBlock.getDictionary().getLogicalSizeInBytes(), 95);
    // The 100 positions in the dictionary block index to 10 positions in the underlying dictionary (10 each).
    // Logical size calculation accounts for 4 bytes of offset and 1 byte of isNull. Therefore the expected unoptimized
    // size is 10 times the size of the underlying dictionary (VariableWidthBlock).
    assertEquals(dictionaryBlock.getLogicalSizeInBytes(), 95 * 10);
    // With alternating nulls, we have 21 positions, with the same size calculation as above.
    dictionaryBlock = createDictionaryBlock(alternatingNullValues(expectedValues), 210);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 21);
    assertEquals(dictionaryBlock.getDictionary().getLogicalSizeInBytes(), 150);
    // The null positions should be included in the logical size.
    assertEquals(dictionaryBlock.getLogicalSizeInBytes(), 150 * 10);
}

Example 60 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestGroupByHash method testMemoryReservationYieldWithDictionary.

@Test
public void testMemoryReservationYieldWithDictionary() {
    // Create a page with positionCount >> expected size of groupByHash
    int dictionaryLength = 1_000;
    int length = 2_000_000;
    int[] ids = IntStream.range(0, dictionaryLength).toArray();
    DictionaryId dictionaryId = randomDictionaryId();
    Block valuesBlock = new DictionaryBlock(dictionaryLength, createStringSequenceBlock(0, length), ids, dictionaryId);
    Block hashBlock = new DictionaryBlock(dictionaryLength, getHashBlock(ImmutableList.of(VARCHAR), valuesBlock), ids, dictionaryId);
    Page page = new Page(valuesBlock, hashBlock);
    AtomicInteger currentQuota = new AtomicInteger(0);
    AtomicInteger allowedQuota = new AtomicInteger(3);
    UpdateMemory updateMemory = () -> {
        if (currentQuota.get() < allowedQuota.get()) {
            currentQuota.getAndIncrement();
            return true;
        }
        return false;
    };
    int yields = 0;
    // test addPage
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    boolean finish = false;
    Work<?> addPageWork = groupByHash.addPage(page);
    while (!finish) {
        finish = addPageWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(addPageWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
    // test getGroupIds
    currentQuota.set(0);
    allowedQuota.set(3);
    yields = 0;
    groupByHash = createGroupByHash(ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 1, true, JOIN_COMPILER, updateMemory);
    finish = false;
    Work<GroupByIdBlock> getGroupIdsWork = groupByHash.getGroupIds(page);
    while (!finish) {
        finish = getGroupIdsWork.process();
        if (!finish) {
            assertEquals(currentQuota.get(), allowedQuota.get());
            // assert if we are blocked, we are going to be blocked again without changing allowedQuota
            assertFalse(getGroupIdsWork.process());
            assertEquals(currentQuota.get(), allowedQuota.get());
            yields++;
            allowedQuota.getAndAdd(3);
        }
    }
    // assert there is not anything missing
    assertEquals(dictionaryLength, groupByHash.getGroupCount());
    assertEquals(dictionaryLength, getGroupIdsWork.getResult().getPositionCount());
    // assert we yield for every 3 rehashes
    // currentQuota is essentially the count we have successfully rehashed
    // the rehash count is 10 = log2(1_000 / 0.75)
    assertEquals(currentQuota.get(), 10);
    assertEquals(currentQuota.get() / 3, yields);
}