Search in sources :

Example 1 with NEW

use of com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation.NEW in project presto by prestodb.

the class TestHistogram method testManyValuesInducingRehash.

private void testManyValuesInducingRehash(InternalAggregationFunction aggregationFunction) {
    double distinctFraction = 0.1f;
    int numGroups = 50000;
    int itemCount = 30;
    Random random = new Random();
    GroupedAccumulator groupedAccumulator = createGroupedAccumulator(aggregationFunction);
    for (int j = 0; j < numGroups; j++) {
        Map<String, Long> expectedValues = new HashMap<>();
        List<String> valueList = new ArrayList<>();
        for (int i = 0; i < itemCount; i++) {
            String str = String.valueOf(i % 10);
            String item = IntStream.range(0, itemCount).mapToObj(x -> str).collect(Collectors.joining());
            boolean distinctValue = random.nextDouble() < distinctFraction;
            if (distinctValue) {
                // produce a unique value for the histogram
                item = j + "-" + item;
                valueList.add(item);
            } else {
                valueList.add(item);
            }
            expectedValues.compute(item, (k, v) -> v == null ? 1L : ++v);
        }
        Block block = createStringsBlock(valueList);
        AggregationTestInputBuilder testInputBuilder = new AggregationTestInputBuilder(new Block[] { block }, aggregationFunction);
        AggregationTestInput test1 = testInputBuilder.build();
        test1.runPagesOnAccumulatorWithAssertion(j, groupedAccumulator, new AggregationTestOutput(expectedValues));
    }
}
Also used : FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) DateTimeZone(org.joda.time.DateTimeZone) MetadataManager(com.facebook.presto.metadata.MetadataManager) Test(org.testng.annotations.Test) Random(java.util.Random) TimeZoneKey.getTimeZoneKey(com.facebook.presto.common.type.TimeZoneKey.getTimeZoneKey) Map(java.util.Map) BlockAssertions.createStringArraysBlock(com.facebook.presto.block.BlockAssertions.createStringArraysBlock) StructuralTestUtil.mapBlockOf(com.facebook.presto.util.StructuralTestUtil.mapBlockOf) NEW(com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation.NEW) DateTimeZoneIndex.getDateTimeZone(com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone) Ints(org.testng.internal.collections.Ints) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) HistogramGroupImplementation(com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation) OperatorAssertion.toRow(com.facebook.presto.operator.OperatorAssertion.toRow) Collectors(java.util.stream.Collectors) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) List(java.util.List) BlockAssertions.createBooleansBlock(com.facebook.presto.block.BlockAssertions.createBooleansBlock) AggregationTestInputBuilder(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestInputBuilder) BlockAssertions.createDoublesBlock(com.facebook.presto.block.BlockAssertions.createDoublesBlock) Optional(java.util.Optional) UpdateMemory(com.facebook.presto.operator.UpdateMemory) AggregationTestUtils.assertAggregation(com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation) IntStream(java.util.stream.IntStream) MapType(com.facebook.presto.common.type.MapType) BlockAssertions.createLongsBlock(com.facebook.presto.block.BlockAssertions.createLongsBlock) TIMESTAMP_WITH_TIME_ZONE(com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) HashMap(java.util.HashMap) TypeSignatureProvider.fromTypes(com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes) ArrayList(java.util.ArrayList) GroupByAggregationTestUtils(com.facebook.presto.operator.aggregation.groupByAggregations.GroupByAggregationTestUtils) AggregationTestOutput(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestOutput) ImmutableList(com.google.common.collect.ImmutableList) NAME(com.facebook.presto.operator.aggregation.histogram.Histogram.NAME) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) StructuralTestUtil.mapType(com.facebook.presto.util.StructuralTestUtil.mapType) AggregationTestInput(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestInput) Type(com.facebook.presto.common.type.Type) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) SqlTimestampWithTimeZone(com.facebook.presto.common.type.SqlTimestampWithTimeZone) DateTime(org.joda.time.DateTime) BlockAssertions.createStringsBlock(com.facebook.presto.block.BlockAssertions.createStringsBlock) TimeZoneKey(com.facebook.presto.common.type.TimeZoneKey) DateTimeEncoding.packDateTimeWithZone(com.facebook.presto.common.type.DateTimeEncoding.packDateTimeWithZone) Assert.assertTrue(org.testng.Assert.assertTrue) Block(com.facebook.presto.common.block.Block) RowType(com.facebook.presto.common.type.RowType) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Random(java.util.Random) AggregationTestOutput(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestOutput) BlockAssertions.createStringArraysBlock(com.facebook.presto.block.BlockAssertions.createStringArraysBlock) BlockAssertions.createBooleansBlock(com.facebook.presto.block.BlockAssertions.createBooleansBlock) BlockAssertions.createDoublesBlock(com.facebook.presto.block.BlockAssertions.createDoublesBlock) BlockAssertions.createLongsBlock(com.facebook.presto.block.BlockAssertions.createLongsBlock) BlockAssertions.createStringsBlock(com.facebook.presto.block.BlockAssertions.createStringsBlock) Block(com.facebook.presto.common.block.Block) AggregationTestInput(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestInput) AggregationTestInputBuilder(com.facebook.presto.operator.aggregation.groupByAggregations.AggregationTestInputBuilder)

Aggregations

BlockAssertions.createBooleansBlock (com.facebook.presto.block.BlockAssertions.createBooleansBlock)1 BlockAssertions.createDoublesBlock (com.facebook.presto.block.BlockAssertions.createDoublesBlock)1 BlockAssertions.createLongsBlock (com.facebook.presto.block.BlockAssertions.createLongsBlock)1 BlockAssertions.createStringArraysBlock (com.facebook.presto.block.BlockAssertions.createStringArraysBlock)1 BlockAssertions.createStringsBlock (com.facebook.presto.block.BlockAssertions.createStringsBlock)1 Block (com.facebook.presto.common.block.Block)1 BlockBuilder (com.facebook.presto.common.block.BlockBuilder)1 ArrayType (com.facebook.presto.common.type.ArrayType)1 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)1 BOOLEAN (com.facebook.presto.common.type.BooleanType.BOOLEAN)1 DateTimeEncoding.packDateTimeWithZone (com.facebook.presto.common.type.DateTimeEncoding.packDateTimeWithZone)1 DOUBLE (com.facebook.presto.common.type.DoubleType.DOUBLE)1 MapType (com.facebook.presto.common.type.MapType)1 RowType (com.facebook.presto.common.type.RowType)1 SqlTimestampWithTimeZone (com.facebook.presto.common.type.SqlTimestampWithTimeZone)1 TimeZoneKey (com.facebook.presto.common.type.TimeZoneKey)1 TimeZoneKey.getTimeZoneKey (com.facebook.presto.common.type.TimeZoneKey.getTimeZoneKey)1 TIMESTAMP_WITH_TIME_ZONE (com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)1 Type (com.facebook.presto.common.type.Type)1 VARCHAR (com.facebook.presto.common.type.VarcharType.VARCHAR)1