Examples with RowPagesBuilder io.trino.RowPagesBuilder used on opensource projects

Example 16 with RowPagesBuilder

use of io.trino.RowPagesBuilder in project trino by trinodb.

the class TestHashAggregationOperator method testHashAggregation.

@Test(dataProvider = "hashEnabledAndMemoryLimitForMergeValues")
public void testHashAggregation(boolean hashEnabled, boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimitForMerge, long memoryLimitForMergeWithMemory) {
    // make operator produce multiple pages during finish phase
    int numberOfRows = 40_000;
    TestingAggregationFunction countVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("count"), fromTypes(VARCHAR));
    TestingAggregationFunction countBooleanColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("count"), fromTypes(BOOLEAN));
    TestingAggregationFunction maxVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("max"), fromTypes(VARCHAR));
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BOOLEAN);
    List<Page> input = rowPagesBuilder.addSequencePage(numberOfRows, 100, 0, 100_000, 0, 500).addSequencePage(numberOfRows, 100, 0, 200_000, 0, 500).addSequencePage(numberOfRows, 100, 0, 300_000, 0, 500).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR), hashChannels, ImmutableList.of(), SINGLE, false, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_SUM.createAggregatorFactory(SINGLE, ImmutableList.of(3), OptionalInt.empty()), LONG_AVERAGE.createAggregatorFactory(SINGLE, ImmutableList.of(3), OptionalInt.empty()), maxVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty()), countVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), countBooleanColumn.createAggregatorFactory(SINGLE, ImmutableList.of(4), OptionalInt.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(DataSize.of(16, MEGABYTE)), spillEnabled, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, blockTypeOperators, Optional.empty());
    DriverContext driverContext = createDriverContext(memoryLimitForMerge);
    MaterializedResult.Builder expectedBuilder = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT);
    for (int i = 0; i < numberOfRows; ++i) {
        expectedBuilder.row(Integer.toString(i), 3L, 3L * i, (double) i, Integer.toString(300_000 + i), 3L, 3L);
    }
    MaterializedResult expected = expectedBuilder.build();
    List<Page> pages = toPages(operatorFactory, driverContext, input, revokeMemoryWhenAddingPages);
    assertGreaterThan(pages.size(), 1, "Expected more than one output page");
    assertPagesEqualIgnoreOrder(driverContext, pages, expected, hashEnabled, Optional.of(hashChannels.size()));
    assertTrue(spillEnabled == (spillerFactory.getSpillsCount() > 0), format("Spill state mismatch. Expected spill: %s, spill count: %s", spillEnabled, spillerFactory.getSpillsCount()));
}

Example 17 with RowPagesBuilder

use of io.trino.RowPagesBuilder in project trino by trinodb.

the class TestHashAggregationOperator method testMultiSliceAggregationOutput.

@Test(dataProvider = "hashEnabled")
public void testMultiSliceAggregationOutput(boolean hashEnabled) {
    // estimate the number of entries required to create 1.5 pages of results
    // See InMemoryHashAggregationBuilder.buildTypes()
    int fixedWidthSize = // Used by BigintGroupByHash, see BigintGroupByHash.TYPES_WITH_RAW_HASH
    SIZE_OF_LONG + SIZE_OF_LONG + SIZE_OF_LONG + // Used by COUNT and LONG_AVERAGE aggregators;
    SIZE_OF_DOUBLE;
    int multiSlicePositionCount = (int) (1.5 * PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES / fixedWidthSize);
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, BIGINT, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(multiSlicePositionCount, 0, 0).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), SINGLE, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_AVERAGE.createAggregatorFactory(SINGLE, ImmutableList.of(1), OptionalInt.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, blockTypeOperators, Optional.empty());
    assertEquals(toPages(operatorFactory, createDriverContext(), input).size(), 2);
}

Example 18 with RowPagesBuilder

use of io.trino.RowPagesBuilder in project trino by trinodb.

the class TestHashAggregationOperator method testMemoryLimit.

@Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node memory limit of 10B.*")
public void testMemoryLimit(boolean hashEnabled) {
    TestingAggregationFunction maxVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("max"), fromTypes(VARCHAR));
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR, BIGINT, VARCHAR, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, DataSize.ofBytes(10)).addPipelineContext(0, true, true, false).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), SINGLE, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_MIN.createAggregatorFactory(SINGLE, ImmutableList.of(3), OptionalInt.empty()), LONG_AVERAGE.createAggregatorFactory(SINGLE, ImmutableList.of(3), OptionalInt.empty()), maxVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, blockTypeOperators, Optional.empty());
    toPages(operatorFactory, driverContext, input);
}

Example 19 with RowPagesBuilder

use of io.trino.RowPagesBuilder in project trino by trinodb.

the class TestHashAggregationOperator method testHashBuilderResize.

@Test(dataProvider = "hashEnabledAndMemoryLimitForMergeValues")
public void testHashBuilderResize(boolean hashEnabled, boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimitForMerge, long memoryLimitForMergeWithMemory) {
    BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1, MAX_BLOCK_SIZE_IN_BYTES);
    // this must be larger than MAX_BLOCK_SIZE_IN_BYTES, 64K
    VARCHAR.writeSlice(builder, Slices.allocate(200_000));
    builder.build();
    List<Integer> hashChannels = Ints.asList(0);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100).addBlocksPage(builder.build()).addSequencePage(10, 100).build();
    DriverContext driverContext = createDriverContext(memoryLimitForMerge);
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR), hashChannels, ImmutableList.of(), SINGLE, false, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(DataSize.of(16, MEGABYTE)), spillEnabled, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, blockTypeOperators, Optional.empty());
    toPages(operatorFactory, driverContext, input, revokeMemoryWhenAddingPages);
}

Example 20 with RowPagesBuilder

use of io.trino.RowPagesBuilder in project trino by trinodb.

the class TestHashSemiJoinOperator method testSemiJoinOnVarcharType.

@Test(dataProvider = "hashEnabledValues")
public void testSemiJoinOnVarcharType(boolean hashEnabled) {
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // build
    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), VARCHAR);
    Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder.row("10").row("30").row("30").row("35").row("36").row("37").row("50").build());
    SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), rowPagesBuilder.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(typeOperators), blockTypeOperators);
    Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
    Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator);
    while (!driver.isFinished()) {
        driver.process();
    }
    // probe
    List<Type> probeTypes = ImmutableList.of(VARCHAR, BIGINT);
    RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(hashEnabled, Ints.asList(0), VARCHAR, BIGINT);
    List<Page> probeInput = rowPagesBuilderProbe.addSequencePage(10, 30, 0).build();
    Optional<Integer> probeHashChannel = hashEnabled ? Optional.of(probeTypes.size()) : Optional.empty();
    OperatorFactory joinOperatorFactory = HashSemiJoinOperator.createOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0, probeHashChannel);
    // expected
    MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row("30", 0L, true).row("31", 1L, false).row("32", 2L, false).row("33", 3L, false).row("34", 4L, false).row("35", 5L, true).row("36", 6L, true).row("37", 7L, true).row("38", 8L, false).row("39", 9L, false).build();
    OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size()));
}