Search in sources :

Example 1 with AggregationTestInputBuilder

use of io.trino.operator.aggregation.groupby.AggregationTestInputBuilder in project trino by trinodb.

the class TestMultimapAggAggregation method testMultimapAggWithGroupBy.

private static <K, V> void testMultimapAggWithGroupBy(TestingAggregationFunction aggregationFunction, GroupedAggregator groupedAggregator, int groupId, Type keyType, List<K> expectedKeys, Type valueType, List<V> expectedValues) {
    RowPageBuilder pageBuilder = RowPageBuilder.rowPageBuilder(keyType, valueType);
    ImmutableMultimap.Builder<K, V> outputBuilder = ImmutableMultimap.builder();
    for (int i = 0; i < expectedValues.size(); i++) {
        pageBuilder.row(expectedKeys.get(i), expectedValues.get(i));
        outputBuilder.put(expectedKeys.get(i), expectedValues.get(i));
    }
    Page page = pageBuilder.build();
    AggregationTestInput input = new AggregationTestInputBuilder(new Block[] { page.getBlock(0), page.getBlock(1) }, aggregationFunction).build();
    AggregationTestOutput testOutput = new AggregationTestOutput(outputBuilder.build().asMap());
    input.runPagesOnAggregatorWithAssertion(groupId, aggregationFunction.getFinalType(), groupedAggregator, testOutput);
}
Also used : AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) RowPageBuilder(io.trino.RowPageBuilder) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) Block(io.trino.spi.block.Block) Page(io.trino.spi.Page) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder)

Example 2 with AggregationTestInputBuilder

use of io.trino.operator.aggregation.groupby.AggregationTestInputBuilder in project trino by trinodb.

the class TestHistogram method testManyValuesInducingRehash.

private static void testManyValuesInducingRehash(TestingAggregationFunction aggregationFunction) {
    double distinctFraction = 0.1f;
    int numGroups = 50000;
    int itemCount = 30;
    Random random = new Random();
    GroupedAggregator groupedAggregator = aggregationFunction.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()).createGroupedAggregator();
    for (int j = 0; j < numGroups; j++) {
        Map<String, Long> expectedValues = new HashMap<>();
        List<String> valueList = new ArrayList<>();
        for (int i = 0; i < itemCount; i++) {
            String str = String.valueOf(i % 10);
            String item = IntStream.range(0, itemCount).mapToObj(x -> str).collect(Collectors.joining());
            boolean distinctValue = random.nextDouble() < distinctFraction;
            if (distinctValue) {
                // produce a unique value for the histogram
                item = j + "-" + item;
                valueList.add(item);
            } else {
                valueList.add(item);
            }
            expectedValues.compute(item, (k, v) -> v == null ? 1L : ++v);
        }
        Block block = createStringsBlock(valueList);
        AggregationTestInputBuilder testInputBuilder = new AggregationTestInputBuilder(new Block[] { block }, aggregationFunction);
        AggregationTestInput test1 = testInputBuilder.build();
        test1.runPagesOnAggregatorWithAssertion(j, aggregationFunction.getFinalType(), groupedAggregator, new AggregationTestOutput(expectedValues));
    }
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) TypeSignatureProvider.fromTypes(io.trino.sql.analyzer.TypeSignatureProvider.fromTypes) TestingFunctionResolution(io.trino.metadata.TestingFunctionResolution) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) Test(org.testng.annotations.Test) Random(java.util.Random) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder) Block(io.trino.spi.block.Block) DateTimeEncoding.unpackZoneKey(io.trino.spi.type.DateTimeEncoding.unpackZoneKey) Map(java.util.Map) TIMESTAMP_WITH_TIME_ZONE(io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) RowType(io.trino.spi.type.RowType) ImmutableMap(com.google.common.collect.ImmutableMap) BlockAssertions.createDoublesBlock(io.trino.block.BlockAssertions.createDoublesBlock) OperatorAssertion.toRow(io.trino.operator.OperatorAssertion.toRow) DateTimeEncoding.packDateTimeWithZone(io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone) ArrayType(io.trino.spi.type.ArrayType) Collectors(java.util.stream.Collectors) SqlTimestampWithTimeZone(io.trino.spi.type.SqlTimestampWithTimeZone) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) DateTimeEncoding.unpackMillisUtc(io.trino.spi.type.DateTimeEncoding.unpackMillisUtc) AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) IntStream(java.util.stream.IntStream) BOOLEAN(io.trino.spi.type.BooleanType.BOOLEAN) SINGLE(io.trino.sql.planner.plan.AggregationNode.Step.SINGLE) HashMap(java.util.HashMap) OptionalInt(java.util.OptionalInt) StructuralTestUtil.mapBlockOf(io.trino.util.StructuralTestUtil.mapBlockOf) ArrayList(java.util.ArrayList) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TimeZoneKey(io.trino.spi.type.TimeZoneKey) Histogram(io.trino.operator.aggregation.histogram.Histogram) DateTimeZoneIndex.getDateTimeZone(io.trino.util.DateTimeZoneIndex.getDateTimeZone) BlockAssertions.createStringsBlock(io.trino.block.BlockAssertions.createStringsBlock) MapType(io.trino.spi.type.MapType) BlockAssertions.createBooleansBlock(io.trino.block.BlockAssertions.createBooleansBlock) DateTime(org.joda.time.DateTime) Ints(com.google.common.primitives.Ints) BlockAssertions.createStringArraysBlock(io.trino.block.BlockAssertions.createStringArraysBlock) QualifiedName(io.trino.sql.tree.QualifiedName) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) TimeZoneKey.getTimeZoneKey(io.trino.spi.type.TimeZoneKey.getTimeZoneKey) StructuralTestUtil.mapType(io.trino.util.StructuralTestUtil.mapType) AggregationTestUtils.assertAggregation(io.trino.operator.aggregation.AggregationTestUtils.assertAggregation) Assert.assertTrue(org.testng.Assert.assertTrue) BlockBuilder(io.trino.spi.block.BlockBuilder) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Random(java.util.Random) AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) Block(io.trino.spi.block.Block) BlockAssertions.createDoublesBlock(io.trino.block.BlockAssertions.createDoublesBlock) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createStringsBlock(io.trino.block.BlockAssertions.createStringsBlock) BlockAssertions.createBooleansBlock(io.trino.block.BlockAssertions.createBooleansBlock) BlockAssertions.createStringArraysBlock(io.trino.block.BlockAssertions.createStringArraysBlock) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder)

Example 3 with AggregationTestInputBuilder

use of io.trino.operator.aggregation.groupby.AggregationTestInputBuilder in project trino by trinodb.

the class TestHistogram method testSharedGroupByWithOverlappingValuesRunner.

private static void testSharedGroupByWithOverlappingValuesRunner(TestingAggregationFunction aggregationFunction) {
    Block block1 = createStringsBlock("a", "b", "c", "d", "a1", "b2", "c3", "d4", "a", "b2", "c", "d4", "a3", "b3", "c3", "b2");
    AggregationTestInputBuilder testInputBuilder1 = new AggregationTestInputBuilder(new Block[] { block1 }, aggregationFunction);
    AggregationTestOutput aggregationTestOutput1 = new AggregationTestOutput(ImmutableMap.<String, Long>builder().put("a", 2L).put("b", 1L).put("c", 2L).put("d", 1L).put("a1", 1L).put("b2", 3L).put("c3", 2L).put("d4", 2L).put("a3", 1L).put("b3", 1L).buildOrThrow());
    AggregationTestInput test1 = testInputBuilder1.build();
    test1.runPagesOnAggregatorWithAssertion(0L, aggregationFunction.getFinalType(), test1.createGroupedAggregator(), aggregationTestOutput1);
}
Also used : AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) Block(io.trino.spi.block.Block) BlockAssertions.createDoublesBlock(io.trino.block.BlockAssertions.createDoublesBlock) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createStringsBlock(io.trino.block.BlockAssertions.createStringsBlock) BlockAssertions.createBooleansBlock(io.trino.block.BlockAssertions.createBooleansBlock) BlockAssertions.createStringArraysBlock(io.trino.block.BlockAssertions.createStringArraysBlock) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder)

Example 4 with AggregationTestInputBuilder

use of io.trino.operator.aggregation.groupby.AggregationTestInputBuilder in project trino by trinodb.

the class TestArrayAggregation method testMultipleGroupsWithMultiplePages.

@Test
public void testMultipleGroupsWithMultiplePages() {
    TestingAggregationFunction varcharAgg = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("array_agg"), fromTypes(VARCHAR));
    Block block1 = createStringsBlock("a", "b", "c", "d", "e");
    Block block2 = createStringsBlock("f", "g", "h", "i", "j");
    AggregationTestOutput aggregationTestOutput1 = new AggregationTestOutput(ImmutableList.of("a", "b", "c", "d", "e"));
    AggregationTestInputBuilder testInputBuilder1 = new AggregationTestInputBuilder(new Block[] { block1 }, varcharAgg);
    AggregationTestInput test1 = testInputBuilder1.build();
    GroupedAggregator groupedAggregator = test1.createGroupedAggregator();
    test1.runPagesOnAggregatorWithAssertion(0L, varcharAgg.getFinalType(), groupedAggregator, aggregationTestOutput1);
    AggregationTestOutput aggregationTestOutput2 = new AggregationTestOutput(ImmutableList.of("f", "g", "h", "i", "j"));
    AggregationTestInputBuilder testBuilder2 = new AggregationTestInputBuilder(new Block[] { block2 }, varcharAgg);
    AggregationTestInput test2 = testBuilder2.build();
    test2.runPagesOnAggregatorWithAssertion(255L, varcharAgg.getFinalType(), groupedAggregator, aggregationTestOutput2);
}
Also used : AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) Block(io.trino.spi.block.Block) BlockAssertions.createStringsBlock(io.trino.block.BlockAssertions.createStringsBlock) BlockAssertions.createBooleansBlock(io.trino.block.BlockAssertions.createBooleansBlock) BlockAssertions.createTypedLongsBlock(io.trino.block.BlockAssertions.createTypedLongsBlock) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createArrayBigintBlock(io.trino.block.BlockAssertions.createArrayBigintBlock) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder) Test(org.testng.annotations.Test)

Example 5 with AggregationTestInputBuilder

use of io.trino.operator.aggregation.groupby.AggregationTestInputBuilder in project trino by trinodb.

the class TestArrayAggregation method testManyValues.

@Test
public void testManyValues() {
    // Test many values so multiple BlockBuilders will be used to store group state.
    TestingAggregationFunction varcharAgg = FUNCTION_RESOLUTION.getAggregateFunction(QualifiedName.of("array_agg"), fromTypes(VARCHAR));
    int numGroups = 50000;
    int arraySize = 30;
    Random random = new Random();
    GroupedAggregator groupedAggregator = varcharAgg.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()).createGroupedAggregator();
    for (int j = 0; j < numGroups; j++) {
        List<String> expectedValues = new ArrayList<>();
        List<String> valueList = new ArrayList<>();
        for (int i = 0; i < arraySize; i++) {
            String str = String.valueOf(random.nextInt());
            valueList.add(str);
            expectedValues.add(str);
        }
        Block block = createStringsBlock(valueList);
        AggregationTestInputBuilder testInputBuilder = new AggregationTestInputBuilder(new Block[] { block }, varcharAgg);
        AggregationTestInput test1 = testInputBuilder.build();
        test1.runPagesOnAggregatorWithAssertion(j, varcharAgg.getFinalType(), groupedAggregator, new AggregationTestOutput(expectedValues));
    }
}
Also used : Random(java.util.Random) AggregationTestOutput(io.trino.operator.aggregation.groupby.AggregationTestOutput) ArrayList(java.util.ArrayList) Block(io.trino.spi.block.Block) BlockAssertions.createStringsBlock(io.trino.block.BlockAssertions.createStringsBlock) BlockAssertions.createBooleansBlock(io.trino.block.BlockAssertions.createBooleansBlock) BlockAssertions.createTypedLongsBlock(io.trino.block.BlockAssertions.createTypedLongsBlock) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createArrayBigintBlock(io.trino.block.BlockAssertions.createArrayBigintBlock) AggregationTestInput(io.trino.operator.aggregation.groupby.AggregationTestInput) AggregationTestInputBuilder(io.trino.operator.aggregation.groupby.AggregationTestInputBuilder) Test(org.testng.annotations.Test)

Aggregations

AggregationTestInput (io.trino.operator.aggregation.groupby.AggregationTestInput)8 AggregationTestInputBuilder (io.trino.operator.aggregation.groupby.AggregationTestInputBuilder)8 AggregationTestOutput (io.trino.operator.aggregation.groupby.AggregationTestOutput)8 Block (io.trino.spi.block.Block)7 BlockAssertions.createBooleansBlock (io.trino.block.BlockAssertions.createBooleansBlock)6 BlockAssertions.createLongsBlock (io.trino.block.BlockAssertions.createLongsBlock)6 BlockAssertions.createStringsBlock (io.trino.block.BlockAssertions.createStringsBlock)6 BlockAssertions.createDoublesBlock (io.trino.block.BlockAssertions.createDoublesBlock)4 BlockAssertions.createStringArraysBlock (io.trino.block.BlockAssertions.createStringArraysBlock)4 Test (org.testng.annotations.Test)4 BlockAssertions.createArrayBigintBlock (io.trino.block.BlockAssertions.createArrayBigintBlock)2 BlockAssertions.createTypedLongsBlock (io.trino.block.BlockAssertions.createTypedLongsBlock)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableMultimap (com.google.common.collect.ImmutableMultimap)1 Ints (com.google.common.primitives.Ints)1 RowPageBuilder (io.trino.RowPageBuilder)1 TestingFunctionResolution (io.trino.metadata.TestingFunctionResolution)1 OperatorAssertion.toRow (io.trino.operator.OperatorAssertion.toRow)1 AggregationTestUtils.assertAggregation (io.trino.operator.aggregation.AggregationTestUtils.assertAggregation)1