Examples with DictionaryBlock io.trino.spi.block.DictionaryBlock used on opensource projects

Example 21 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestArrayOfRowsUnnester method testArrayOfRowsUnnester.

/**
 * Test operations of ArrayOfRowUnnester incrementally on the input.
 * Output final blocks after the whole input has been processed.
 * <p>
 * Input 3d array {@code elements} stores values from a column with type <array<row<varchar, varchar, ... {@code fieldCount} times> >.
 * elements[i] corresponds to a position in this column, represents one array of row(....).
 * elements[i][j] represents one row(....) object in the array.
 * elements[i][j][k] represents value of kth field in row(...) object.
 */
private static Block[] testArrayOfRowsUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements, int fieldCount) {
    validateTestInput(requiredOutputCounts, unnestedLengths, elements, fieldCount);
    int positionCount = requiredOutputCounts.length;
    // True if there is a null Row element inside the array at this position
    boolean[] containsNullRowElement = new boolean[positionCount];
    // Populate containsNullRowElement
    for (int i = 0; i < positionCount; i++) {
        containsNullRowElement[i] = false;
        if (elements[i] != null) {
            for (int j = 0; j < elements[i].length; j++) {
                if (elements[i][j] == null) {
                    containsNullRowElement[i] = true;
                }
            }
        }
    }
    // Check for null elements in individual input fields
    boolean[] nullsPresent = new boolean[fieldCount];
    for (int field = 0; field < fieldCount; field++) {
        nullsPresent[field] = nullExists(elements[field]);
    }
    // Create the unnester and input block
    RowType rowType = RowType.anonymous(Collections.nCopies(fieldCount, VARCHAR));
    Unnester arrayofRowsUnnester = new ArrayOfRowsUnnester(rowType);
    Block arrayBlockOfRows = createArrayBlockOfRowBlocks(elements, rowType);
    Block[] blocks = null;
    // Verify output being produced after processing every position. (quadratic)
    for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) {
        // Reset input and prepare for new output
        PageBuilderStatus status = new PageBuilderStatus();
        arrayofRowsUnnester.resetInput(arrayBlockOfRows);
        assertEquals(arrayofRowsUnnester.getInputEntryCount(), elements.length);
        arrayofRowsUnnester.startNewOutput(status, 10);
        boolean misAligned = false;
        // Process inputTestCount positions
        for (int i = 0; i < inputTestCount; i++) {
            arrayofRowsUnnester.processCurrentAndAdvance(requiredOutputCounts[i]);
            int elementsSize = (elements[i] != null ? elements[i].length : 0);
            // (2) null Row element
            if ((requiredOutputCounts[i] > elementsSize) || containsNullRowElement[i]) {
                misAligned = true;
            }
        }
        // Build output block and verify
        blocks = arrayofRowsUnnester.buildOutputBlocksAndFlush();
        assertEquals(blocks.length, rowType.getFields().size());
        // Verify output blocks for individual fields
        for (int field = 0; field < blocks.length; field++) {
            assertTrue((blocks[field] instanceof DictionaryBlock) || (!nullsPresent[field] && misAligned));
            assertFalse((blocks[field] instanceof DictionaryBlock) && (!nullsPresent[field] && misAligned));
            Slice[][] fieldElements = getFieldElements(elements, field);
            Slice[] expectedOutput = computeExpectedUnnestedOutput(fieldElements, requiredOutputCounts, 0, inputTestCount);
            assertBlock(blocks[field], expectedOutput);
        }
    }
    return blocks;
}

Example 22 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestArrayUnnester method testArrayUnnester.

private static Block[] testArrayUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements) {
    Slice[][] slices = getFieldElements(elements, 0);
    validateTestInput(requiredOutputCounts, unnestedLengths, elements, 1);
    // Check if there is a null element in the input
    boolean nullPresent = nullExists(slices);
    // Initialize unnester
    Unnester arrayUnnester = new ArrayUnnester(VARCHAR);
    Block arrayBlock = createArrayBlock(slices);
    Block[] blocks = null;
    // Verify output being produced after processing every position. (quadratic)
    for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) {
        // Reset input
        arrayUnnester.resetInput(arrayBlock);
        assertEquals(arrayUnnester.getInputEntryCount(), elements.length);
        // Prepare for new output
        PageBuilderStatus status = new PageBuilderStatus();
        arrayUnnester.startNewOutput(status, 10);
        boolean misAligned = false;
        // Process inputTestCount positions
        for (int i = 0; i < inputTestCount; i++) {
            int elementsSize = (elements[i] != null ? elements[i].length : 0);
            assertEquals(arrayUnnester.getCurrentUnnestedLength(), elementsSize);
            arrayUnnester.processCurrentAndAdvance(requiredOutputCounts[i]);
            if (requiredOutputCounts[i] > elementsSize) {
                misAligned = true;
            }
        }
        // Verify output
        blocks = arrayUnnester.buildOutputBlocksAndFlush();
        assertEquals(blocks.length, 1);
        assertTrue((blocks[0] instanceof DictionaryBlock) || (!nullPresent && misAligned));
        assertFalse((blocks[0] instanceof DictionaryBlock) && (!nullPresent && misAligned));
        Slice[] expectedOutput = computeExpectedUnnestedOutput(slices, requiredOutputCounts, 0, inputTestCount);
        assertBlock(blocks[0], expectedOutput);
    }
    return blocks;
}

Example 23 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestPageProcessorCompiler method testSanityColumnarDictionary.

@Test
public void testSanityColumnarDictionary() {
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
}

Example 24 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestPageProcessorCompiler method testNonDeterministicProject.

@Test
public void testNonDeterministicProject() {
    ResolvedFunction lessThan = functionResolution.resolveOperator(LESS_THAN, ImmutableList.of(BIGINT, BIGINT));
    CallExpression random = new CallExpression(functionResolution.resolveFunction(QualifiedName.of("random"), fromTypes(BIGINT)), singletonList(constant(10L, BIGINT)));
    InputReferenceExpression col0 = field(0, BIGINT);
    CallExpression lessThanRandomExpression = new CallExpression(lessThan, ImmutableList.of(col0, random));
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(lessThanRandomExpression), MAX_BATCH_SIZE).get();
    assertFalse(isDeterministic(lessThanRandomExpression));
    Page page = new Page(createLongDictionaryBlock(1, 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertFalse(outputPage.getBlock(0) instanceof DictionaryBlock);
}

Example 25 with DictionaryBlock

use of io.trino.spi.block.DictionaryBlock in project trino by trinodb.

the class TestReplicatedBlockBuilder method testReplication.

private static void testReplication(Slice[] values, int[] counts) {
    assertEquals(values.length, counts.length);
    ReplicatedBlockBuilder replicateBlockBuilder = new ReplicatedBlockBuilder();
    Block valuesBlock = createSimpleBlock(values);
    replicateBlockBuilder.resetInputBlock(valuesBlock);
    replicateBlockBuilder.startNewOutput(100);
    for (int i = 0; i < counts.length; i++) {
        replicateBlockBuilder.appendRepeated(i, counts[i]);
    }
    Block outputBlock = replicateBlockBuilder.buildOutputAndFlush();
    assertBlock(outputBlock, createReplicatedOutputSlice(values, counts));
    assertTrue(outputBlock instanceof DictionaryBlock);
}