Examples with PartialAggregationController io.trino.operator.aggregation.partial.PartialAggregationController used on opensource projects

Example 1 with PartialAggregationController

use of io.trino.operator.aggregation.partial.PartialAggregationController in project trino by trinodb.

the class TestHashAggregationOperator method testAdaptivePartialAggregationTriggeredOnlyOnFlush.

@Test
public void testAdaptivePartialAggregationTriggeredOnlyOnFlush() {
    List<Integer> hashChannels = Ints.asList(0);
    PartialAggregationController partialAggregationController = new PartialAggregationController(5, 0.8);
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), PARTIAL, ImmutableList.of(LONG_MIN.createAggregatorFactory(PARTIAL, ImmutableList.of(0), OptionalInt.empty())), Optional.empty(), Optional.empty(), 10, // this setting makes operator to flush only after all pages
    Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, blockTypeOperators, // use 5 rows threshold to trigger adaptive partial aggregation after each page flush
    Optional.of(partialAggregationController));
    List<Page> operator1Input = rowPagesBuilder(false, hashChannels, BIGINT).addSequencePage(10, // first page are unique values, so it would trigger adaptation, but it won't because flush is not called
    0).addBlocksPage(// second page will be hashed to existing value 1
    createRLEBlock(1, 2)).build();
    // the total unique ows ratio for the first operator will be 10/12 so > 0.8 (adaptive partial aggregation uniqueRowsRatioThreshold)
    List<Page> operator1Expected = rowPagesBuilder(BIGINT, BIGINT).addSequencePage(10, 0, // we are expecting second page to be squashed with the first
    0).build();
    assertOperatorEquals(operatorFactory, operator1Input, operator1Expected);
    // the first operator flush disables partial aggregation
    assertTrue(partialAggregationController.isPartialAggregationDisabled());
    // second operator using the same factory, reuses PartialAggregationControl, so it will only produce raw pages (partial aggregation is disabled at this point)
    List<Page> operator2Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10)).build();
    List<Page> operator2Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createRLEBlock(1, 10), createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10), createRLEBlock(2, 10)).build();
    assertOperatorEquals(operatorFactory, operator2Input, operator2Expected);
}

Example 2 with PartialAggregationController

use of io.trino.operator.aggregation.partial.PartialAggregationController in project trino by trinodb.

the class TestHashAggregationOperator method testAdaptivePartialAggregation.

@Test
public void testAdaptivePartialAggregation() {
    List<Integer> hashChannels = Ints.asList(0);
    PartialAggregationController partialAggregationController = new PartialAggregationController(5, 0.8);
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), PARTIAL, ImmutableList.of(LONG_MIN.createAggregatorFactory(PARTIAL, ImmutableList.of(0), OptionalInt.empty())), Optional.empty(), Optional.empty(), 100, // this setting makes operator to flush after each page
    Optional.of(DataSize.ofBytes(1)), joinCompiler, blockTypeOperators, // use 5 rows threshold to trigger adaptive partial aggregation after each page flush
    Optional.of(partialAggregationController));
    // at the start partial aggregation is enabled
    assertFalse(partialAggregationController.isPartialAggregationDisabled());
    // First operator will trigger adaptive partial aggregation after the first page
    List<Page> operator1Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(// first page will be hashed but the values are almost unique, so it will trigger adaptation
    createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8, 8)).addBlocksPage(// second page would be hashed to existing value 1. but if adaptive PA kicks in, the raw values will be passed on
    createRLEBlock(1, 10)).build();
    List<Page> operator1Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8), // the last position was aggregated
    createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8)).addBlocksPage(createRLEBlock(1, 10), // we are expecting second page with raw values
    createRLEBlock(1, 10)).build();
    assertOperatorEquals(operatorFactory, operator1Input, operator1Expected);
    // the first operator flush disables partial aggregation
    assertTrue(partialAggregationController.isPartialAggregationDisabled());
    // second operator using the same factory, reuses PartialAggregationControl, so it will only produce raw pages (partial aggregation is disabled at this point)
    List<Page> operator2Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10)).build();
    List<Page> operator2Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createRLEBlock(1, 10), createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10), createRLEBlock(2, 10)).build();
    assertOperatorEquals(operatorFactory, operator2Input, operator2Expected);
}