Examples with DictionaryBlock io.trino.spi.block.DictionaryBlock used on opensource projects

Example 1 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class MultiChannelGroupByHash method createPageWithExtractedDictionary.

// For a page that contains DictionaryBlocks, create a new page in which
// the dictionaries from the DictionaryBlocks are extracted into the corresponding channels
// From Page(DictionaryBlock1, DictionaryBlock2) create new page with Page(dictionary1, dictionary2)
private Page createPageWithExtractedDictionary(Page page) {
    Block[] blocks = new Block[page.getChannelCount()];
    Block dictionary = ((DictionaryBlock) page.getBlock(channels[0])).getDictionary();
    // extract data dictionary
    blocks[channels[0]] = dictionary;
    // extract hash dictionary
    inputHashChannel.ifPresent(integer -> blocks[integer] = ((DictionaryBlock) page.getBlock(integer)).getDictionary());
    return new Page(dictionary.getPositionCount(), blocks);
}

Example 2 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestPageProcessorCompiler method testSanityFilterOnDictionary.

@Test
public void testSanityFilterOnDictionary() {
    CallExpression lengthVarchar = new CallExpression(functionResolution.resolveFunction(QualifiedName.of("length"), fromTypes(VARCHAR)), ImmutableList.of(field(0, VARCHAR)));
    ResolvedFunction lessThan = functionResolution.resolveOperator(LESS_THAN, ImmutableList.of(BIGINT, BIGINT));
    CallExpression filter = new CallExpression(lessThan, ImmutableList.of(lengthVarchar, constant(10L, BIGINT)));
    PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
    // test filter caching
    Page outputPage2 = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage2.getPositionCount(), 100);
    assertTrue(outputPage2.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock2 = (DictionaryBlock) outputPage2.getBlock(0);
    // both output pages must have the same dictionary
    assertEquals(dictionaryBlock2.getDictionary(), dictionaryBlock.getDictionary());
}

Example 3 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestUnnestBlockBuilder method testEmptyOutput.

@Test
public void testEmptyOutput() {
    Slice[] values = toSlices(new String[] { "a", "b" });
    UnnestBlockBuilder unnestBlockBuilder = new UnnestBlockBuilder(VARCHAR);
    Block valuesBlock = createSimpleBlock(values);
    unnestBlockBuilder.resetInputBlock(valuesBlock);
    unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10);
    // Nothing added to the output
    Block block = unnestBlockBuilder.buildOutputAndFlush();
    assertTrue(block instanceof DictionaryBlock);
    assertBlock(block, new Slice[] {});
}

Example 4 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestMapUnnester method testMapUnnester.

private static Block[] testMapUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements) {
    validateTestInput(requiredOutputCounts, unnestedLengths, elements, 2);
    int positionCount = unnestedLengths.length;
    for (int index = 0; index < positionCount; index++) {
        if (elements[index] != null) {
            for (int i = 0; i < elements[index].length; i++) {
                assertTrue(elements[index][i] != null, "entry cannot be null");
                assertEquals(elements[index][i].length, 2);
            }
        }
    }
    // Check for null elements in individual input fields
    boolean[] nullsPresent = new boolean[2];
    nullsPresent[0] = nullExists(elements[0]);
    nullsPresent[1] = nullExists(elements[1]);
    assertFalse(nullsPresent[0]);
    // Create the unnester and input block
    Unnester mapUnnester = new MapUnnester(VARCHAR, VARCHAR);
    Block mapBlock = createBlockBuilderWithValues(elements).build();
    Block[] blocks = null;
    // Verify output being produced after processing every position. (quadratic)
    for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) {
        // Reset input and prepare for new output
        PageBuilderStatus status = new PageBuilderStatus();
        mapUnnester.resetInput(mapBlock);
        assertEquals(mapUnnester.getInputEntryCount(), elements.length);
        mapUnnester.startNewOutput(status, 10);
        boolean misAligned = false;
        // Process inputTestCount positions
        for (int i = 0; i < inputTestCount; i++) {
            mapUnnester.processCurrentAndAdvance(requiredOutputCounts[i]);
            int elementsSize = (elements[i] != null ? elements[i].length : 0);
            // Check for misalignment, caused by required output count being greater than unnested length
            if ((requiredOutputCounts[i] > elementsSize)) {
                misAligned = true;
            }
        }
        // Build output
        blocks = mapUnnester.buildOutputBlocksAndFlush();
        assertEquals(blocks.length, 2);
        // Verify output blocks for individual fields
        for (int field = 0; field < blocks.length; field++) {
            assertTrue((blocks[field] instanceof DictionaryBlock) || (!nullsPresent[field] && misAligned));
            assertFalse((blocks[field] instanceof DictionaryBlock) && (!nullsPresent[field] && misAligned));
            Slice[][] fieldElements = getFieldElements(elements, field);
            Slice[] expectedOutput = computeExpectedUnnestedOutput(fieldElements, requiredOutputCounts, 0, inputTestCount);
            assertBlock(blocks[field], expectedOutput);
        }
    }
    return blocks;
}

Example 5 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class SortedRangeSet method fromUnorderedValuesBlock.

private static SortedRangeSet fromUnorderedValuesBlock(Type type, Block block) {
    // choice of placing unordered values first or last does not matter for this code
    MethodHandle comparisonOperator = TUPLE_DOMAIN_TYPE_OPERATORS.getComparisonUnorderedLastOperator(type, simpleConvention(FAIL_ON_NULL, BLOCK_POSITION, BLOCK_POSITION));
    List<Integer> indexes = new ArrayList<>(block.getPositionCount());
    for (int position = 0; position < block.getPositionCount(); position++) {
        indexes.add(position);
    }
    indexes.sort((left, right) -> compareValues(comparisonOperator, block, left, block, right));
    int[] dictionary = new int[block.getPositionCount() * 2];
    dictionary[0] = indexes.get(0);
    dictionary[1] = indexes.get(0);
    int dictionaryIndex = 2;
    for (int i = 1; i < indexes.size(); i++) {
        int compare = compareValues(comparisonOperator, block, indexes.get(i - 1), block, indexes.get(i));
        if (compare > 0) {
            throw new IllegalStateException("Values not sorted");
        }
        if (compare == 0) {
            // equal, skip
            continue;
        }
        dictionary[dictionaryIndex] = indexes.get(i);
        dictionaryIndex++;
        dictionary[dictionaryIndex] = indexes.get(i);
        dictionaryIndex++;
    }
    boolean[] inclusive = new boolean[dictionaryIndex];
    Arrays.fill(inclusive, true);
    return new SortedRangeSet(type, inclusive, new DictionaryBlock(dictionaryIndex, block, dictionary));
}