use of io.trino.spi.Page in project trino by trinodb.
the class TestGroupedTopNRankBuilder method testYield.
@Test
public void testYield() {
TypeOperators typeOperators = new TypeOperators();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
Page input = rowPagesBuilder(types).row(1L, 0.3).row(1L, 0.2).row(1L, 0.9).row(1L, 0.1).build().get(0);
input.compact();
AtomicBoolean unblock = new AtomicBoolean();
GroupByHash groupByHash = createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), unblock::get, typeOperators, blockTypeOperators);
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNRankBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST), typeOperators), new SimplePageWithPositionEqualsAndHash(types, ImmutableList.of(1), blockTypeOperators), 5, false, groupByHash);
Work<?> work = groupedTopNBuilder.processPage(input);
assertFalse(work.process());
assertFalse(work.process());
unblock.set(true);
assertTrue(work.process());
List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult());
assertEquals(output.size(), 1);
Page expected = rowPagesBuilder(types).row(1L, 0.1).row(1L, 0.2).row(1L, 0.3).row(1L, 0.9).build().get(0);
assertPageEquals(types, output.get(0), expected);
}
use of io.trino.spi.Page in project trino by trinodb.
the class TestGroupedTopNRankBuilder method testMultiGroupTopN.
@Test(dataProvider = "produceRanking")
public void testMultiGroupTopN(boolean produceRanking) {
TypeOperators typeOperators = new TypeOperators();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
GroupByHash groupByHash = createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), NOOP, typeOperators, blockTypeOperators);
GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNRankBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST), typeOperators), new SimplePageWithPositionEqualsAndHash(types, ImmutableList.of(1), blockTypeOperators), 3, produceRanking, groupByHash);
// Expected effect:
// Group 0 [0.2 x 1 => rank=1, 0.3 x 3 => rank=2]
// Group 1 [0.2 x 1 => rank=1]
assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.3).row(0L, 0.3).row(0L, 0.3).row(0L, 0.2).row(1L, 0.2).build()).process());
// Page should be dropped, because all values too large to be considered
assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.4).row(1L, 0.4).build()).process());
// Next page should cause evict 0.3 from group 0, which should cause the first page to be compacted
// Expected effect:
// Group 0 [0.1 x 1 => rank=1, 0.2 x 2 => rank=2]
// Group 1 [0.2 x 2 => rank=1, 0.3 x 2 => rank=3]
assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.1).row(1L, 0.2).row(0L, 0.3).row(0L, 0.2).row(1L, 0.5).row(1L, 0.4).row(1L, 0.3).row(1L, 0.3).build()).process());
List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult());
assertEquals(output.size(), 1);
List<Type> outputTypes = ImmutableList.of(BIGINT, DOUBLE, BIGINT);
Page expected = rowPageBuilder(outputTypes).row(0, 0.1, 1).row(0, 0.2, 2).row(0, 0.2, 2).row(1, 0.2, 1).row(1, 0.2, 1).row(1, 0.3, 3).row(1, 0.3, 3).build();
if (!produceRanking) {
outputTypes = outputTypes.subList(0, outputTypes.size() - 1);
expected = dropLastColumn(expected);
}
assertPageEquals(outputTypes, getOnlyElement(output), expected);
}
use of io.trino.spi.Page in project trino by trinodb.
the class TestHashAggregationOperator method testAdaptivePartialAggregationTriggeredOnlyOnFlush.
@Test
public void testAdaptivePartialAggregationTriggeredOnlyOnFlush() {
List<Integer> hashChannels = Ints.asList(0);
PartialAggregationController partialAggregationController = new PartialAggregationController(5, 0.8);
HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), PARTIAL, ImmutableList.of(LONG_MIN.createAggregatorFactory(PARTIAL, ImmutableList.of(0), OptionalInt.empty())), Optional.empty(), Optional.empty(), 10, // this setting makes operator to flush only after all pages
Optional.of(DataSize.of(16, MEGABYTE)), joinCompiler, blockTypeOperators, // use 5 rows threshold to trigger adaptive partial aggregation after each page flush
Optional.of(partialAggregationController));
List<Page> operator1Input = rowPagesBuilder(false, hashChannels, BIGINT).addSequencePage(10, // first page are unique values, so it would trigger adaptation, but it won't because flush is not called
0).addBlocksPage(// second page will be hashed to existing value 1
createRLEBlock(1, 2)).build();
// the total unique ows ratio for the first operator will be 10/12 so > 0.8 (adaptive partial aggregation uniqueRowsRatioThreshold)
List<Page> operator1Expected = rowPagesBuilder(BIGINT, BIGINT).addSequencePage(10, 0, // we are expecting second page to be squashed with the first
0).build();
assertOperatorEquals(operatorFactory, operator1Input, operator1Expected);
// the first operator flush disables partial aggregation
assertTrue(partialAggregationController.isPartialAggregationDisabled());
// second operator using the same factory, reuses PartialAggregationControl, so it will only produce raw pages (partial aggregation is disabled at this point)
List<Page> operator2Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10)).build();
List<Page> operator2Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createRLEBlock(1, 10), createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10), createRLEBlock(2, 10)).build();
assertOperatorEquals(operatorFactory, operator2Input, operator2Expected);
}
use of io.trino.spi.Page in project trino by trinodb.
the class TestHashAggregationOperator method testAdaptivePartialAggregation.
@Test
public void testAdaptivePartialAggregation() {
List<Integer> hashChannels = Ints.asList(0);
PartialAggregationController partialAggregationController = new PartialAggregationController(5, 0.8);
HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), PARTIAL, ImmutableList.of(LONG_MIN.createAggregatorFactory(PARTIAL, ImmutableList.of(0), OptionalInt.empty())), Optional.empty(), Optional.empty(), 100, // this setting makes operator to flush after each page
Optional.of(DataSize.ofBytes(1)), joinCompiler, blockTypeOperators, // use 5 rows threshold to trigger adaptive partial aggregation after each page flush
Optional.of(partialAggregationController));
// at the start partial aggregation is enabled
assertFalse(partialAggregationController.isPartialAggregationDisabled());
// First operator will trigger adaptive partial aggregation after the first page
List<Page> operator1Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(// first page will be hashed but the values are almost unique, so it will trigger adaptation
createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8, 8)).addBlocksPage(// second page would be hashed to existing value 1. but if adaptive PA kicks in, the raw values will be passed on
createRLEBlock(1, 10)).build();
List<Page> operator1Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8), // the last position was aggregated
createLongsBlock(0, 1, 2, 3, 4, 5, 6, 7, 8)).addBlocksPage(createRLEBlock(1, 10), // we are expecting second page with raw values
createRLEBlock(1, 10)).build();
assertOperatorEquals(operatorFactory, operator1Input, operator1Expected);
// the first operator flush disables partial aggregation
assertTrue(partialAggregationController.isPartialAggregationDisabled());
// second operator using the same factory, reuses PartialAggregationControl, so it will only produce raw pages (partial aggregation is disabled at this point)
List<Page> operator2Input = rowPagesBuilder(false, hashChannels, BIGINT).addBlocksPage(createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10)).build();
List<Page> operator2Expected = rowPagesBuilder(BIGINT, BIGINT).addBlocksPage(createRLEBlock(1, 10), createRLEBlock(1, 10)).addBlocksPage(createRLEBlock(2, 10), createRLEBlock(2, 10)).build();
assertOperatorEquals(operatorFactory, operator2Input, operator2Expected);
}
use of io.trino.spi.Page in project trino by trinodb.
the class TestHashAggregationOperator method testHashAggregationWithGlobals.
@Test(dataProvider = "hashEnabledAndMemoryLimitForMergeValues")
public void testHashAggregationWithGlobals(boolean hashEnabled, boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimitForMerge, long memoryLimitForMergeWithMemory) {
TestingAggregationFunction countVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("count"), fromTypes(VARCHAR));
TestingAggregationFunction countBooleanColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("count"), fromTypes(BOOLEAN));
TestingAggregationFunction maxVarcharColumn = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("max"), fromTypes(VARCHAR));
Optional<Integer> groupIdChannel = Optional.of(1);
List<Integer> groupByChannels = Ints.asList(1, 2);
List<Integer> globalAggregationGroupIds = Ints.asList(42, 49);
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, groupByChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN);
List<Page> input = rowPagesBuilder.build();
HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), groupByChannels, globalAggregationGroupIds, SINGLE, true, ImmutableList.of(COUNT.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), LONG_MIN.createAggregatorFactory(SINGLE, ImmutableList.of(4), OptionalInt.empty()), LONG_AVERAGE.createAggregatorFactory(SINGLE, ImmutableList.of(4), OptionalInt.empty()), maxVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(2), OptionalInt.empty()), countVarcharColumn.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()), countBooleanColumn.createAggregatorFactory(SINGLE, ImmutableList.of(5), OptionalInt.empty())), rowPagesBuilder.getHashChannel(), groupIdChannel, 100_000, Optional.of(DataSize.of(16, MEGABYTE)), spillEnabled, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, blockTypeOperators, Optional.empty());
DriverContext driverContext = createDriverContext(memoryLimitForMerge);
MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT).row(null, 42L, 0L, null, null, null, 0L, 0L).row(null, 49L, 0L, null, null, null, 0L, 0L).build();
assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected, hashEnabled, Optional.of(groupByChannels.size()), revokeMemoryWhenAddingPages);
}
Aggregations