Examples with DictionaryBlock io.prestosql.spi.block.DictionaryBlock used on opensource projects

Example 1 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class SliceDictionaryColumnWriter method writeDictionaryRowGroup.

private boolean writeDictionaryRowGroup(Block dictionary, int count, IntBigArray dictionaryIndexes, int maxDirectBytes) {
    int valueCount = count;
    int[][] segments = dictionaryIndexes.getSegments();
    for (int i = 0; valueCount > 0 && i < segments.length; i++) {
        int[] segment = segments[i];
        int positionCount = Math.min(valueCount, segment.length);
        Block block = new DictionaryBlock(positionCount, dictionary, segment);
        while (block != null) {
            int chunkPositionCount = block.getPositionCount();
            Block chunk = block.getRegion(0, chunkPositionCount);
            // avoid chunk with huge logical size
            while (chunkPositionCount > 1 && chunk.getLogicalSizeInBytes() > DIRECT_CONVERSION_CHUNK_MAX_LOGICAL_BYTES) {
                chunkPositionCount /= 2;
                chunk = chunk.getRegion(0, chunkPositionCount);
            }
            directColumnWriter.writeBlock(chunk);
            if (directColumnWriter.getBufferedBytes() > maxDirectBytes) {
                return false;
            }
            // slice block to only unconverted rows
            if (chunkPositionCount < block.getPositionCount()) {
                block = block.getRegion(chunkPositionCount, block.getPositionCount() - chunkPositionCount);
            } else {
                block = null;
            }
        }
        valueCount -= positionCount;
    }
    checkState(valueCount == 0);
    return true;
}

Example 2 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestLookupJoinPageBuilder method testDifferentPositions.

@Test
public void testDifferentPositions() {
    int entries = 100;
    BlockBuilder blockBuilder = BIGINT.createFixedSizeBlockBuilder(entries);
    for (int i = 0; i < entries; i++) {
        BIGINT.writeLong(blockBuilder, i);
    }
    Block block = blockBuilder.build();
    Page page = new Page(block);
    JoinProbeFactory joinProbeFactory = new JoinProbeFactory(new int[] { 0 }, ImmutableList.of(0), OptionalInt.empty());
    LookupSource lookupSource = new TestLookupSource(ImmutableList.of(BIGINT), page);
    LookupJoinPageBuilder lookupJoinPageBuilder = new LookupJoinPageBuilder(ImmutableList.of(BIGINT));
    // empty
    JoinProbe probe = joinProbeFactory.createJoinProbe(page);
    Page output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertTrue(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), 0);
    lookupJoinPageBuilder.reset();
    // the probe covers non-sequential positions
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        if (joinPosition % 2 == 1) {
            continue;
        }
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertTrue(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), entries / 2);
    for (int i = 0; i < entries / 2; i++) {
        assertEquals(output.getBlock(0).getLong(i, 0), i * 2);
        assertEquals(output.getBlock(1).getLong(i, 0), i * 2);
    }
    lookupJoinPageBuilder.reset();
    // the probe covers everything
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertFalse(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), entries);
    for (int i = 0; i < entries; i++) {
        assertEquals(output.getBlock(0).getLong(i, 0), i);
        assertEquals(output.getBlock(1).getLong(i, 0), i);
    }
    lookupJoinPageBuilder.reset();
    // the probe covers some sequential positions
    probe = joinProbeFactory.createJoinProbe(page);
    for (int joinPosition = 0; probe.advanceNextPosition(); joinPosition++) {
        if (joinPosition < 10 || joinPosition >= 50) {
            continue;
        }
        lookupJoinPageBuilder.appendRow(probe, lookupSource, joinPosition);
    }
    output = lookupJoinPageBuilder.build(probe);
    assertEquals(output.getChannelCount(), 2);
    assertFalse(output.getBlock(0) instanceof DictionaryBlock);
    assertEquals(output.getPositionCount(), 40);
    for (int i = 10; i < 50; i++) {
        assertEquals(output.getBlock(0).getLong(i - 10, 0), i);
        assertEquals(output.getBlock(1).getLong(i - 10, 0), i);
    }
}

Example 3 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class BlockAssertions method createStringDictionaryBlock.

public static Block createStringDictionaryBlock(int start, int length) {
    checkArgument(length > 5, "block must have more than 5 entries");
    int dictionarySize = length / 5;
    BlockBuilder builder = VARCHAR.createBlockBuilder(null, dictionarySize);
    for (int i = start; i < start + dictionarySize; i++) {
        VARCHAR.writeString(builder, String.valueOf(i));
    }
    int[] ids = new int[length];
    for (int i = 0; i < length; i++) {
        ids[i] = i % dictionarySize;
    }
    return new DictionaryBlock(builder.build(), ids);
}

Example 4 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testCopyPositionsNoCompaction.

@Test
public void testCopyPositionsNoCompaction() {
    Slice[] expectedValues = createExpectedValues(1);
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    int[] positionsToCopy = new int[] { 0, 2, 4, 5 };
    DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length);
    assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length);
    assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, expectedValues);
}

Example 5 with DictionaryBlock

use of io.prestosql.spi.block.DictionaryBlock in project hetu-core by openlookeng.

the class TestDictionaryBlock method testCopyPositionsSamePosition.

@Test
public void testCopyPositionsSamePosition() {
    Slice[] expectedValues = createExpectedValues(10);
    DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100);
    int[] positionsToCopy = new int[] { 52, 52, 52 };
    DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length);
    assertEquals(copiedBlock.getDictionary().getPositionCount(), 1);
    assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length);
    assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[2] });
    assertDictionaryIds(copiedBlock, 0, 0, 0);
}