Examples with Histogram io.trino.operator.aggregation.histogram.Histogram used on opensource projects

Example 1 with Histogram

use of io.trino.operator.aggregation.histogram.Histogram in project trino by trinodb.

the class TestHistogram method testManyValuesInducingRehash.

private static void testManyValuesInducingRehash(TestingAggregationFunction aggregationFunction) {
    double distinctFraction = 0.1f;
    int numGroups = 50000;
    int itemCount = 30;
    Random random = new Random();
    GroupedAggregator groupedAggregator = aggregationFunction.createAggregatorFactory(SINGLE, ImmutableList.of(0), OptionalInt.empty()).createGroupedAggregator();
    for (int j = 0; j < numGroups; j++) {
        Map<String, Long> expectedValues = new HashMap<>();
        List<String> valueList = new ArrayList<>();
        for (int i = 0; i < itemCount; i++) {
            String str = String.valueOf(i % 10);
            String item = IntStream.range(0, itemCount).mapToObj(x -> str).collect(Collectors.joining());
            boolean distinctValue = random.nextDouble() < distinctFraction;
            if (distinctValue) {
                // produce a unique value for the histogram
                item = j + "-" + item;
                valueList.add(item);
            } else {
                valueList.add(item);
            }
            expectedValues.compute(item, (k, v) -> v == null ? 1L : ++v);
        }
        Block block = createStringsBlock(valueList);
        AggregationTestInputBuilder testInputBuilder = new AggregationTestInputBuilder(new Block[] { block }, aggregationFunction);
        AggregationTestInput test1 = testInputBuilder.build();
        test1.runPagesOnAggregatorWithAssertion(j, aggregationFunction.getFinalType(), groupedAggregator, new AggregationTestOutput(expectedValues));
    }
}

Example 2 with Histogram

use of io.trino.operator.aggregation.histogram.Histogram in project trino by trinodb.

the class SystemFunctionBundle method create.

public static FunctionBundle create(FeaturesConfig featuresConfig, TypeOperators typeOperators, BlockTypeOperators blockTypeOperators, NodeVersion nodeVersion) {
    InternalFunctionBundleBuilder builder = InternalFunctionBundle.builder().window(RowNumberFunction.class).window(RankFunction.class).window(DenseRankFunction.class).window(PercentRankFunction.class).window(CumulativeDistributionFunction.class).window(NTileFunction.class).window(FirstValueFunction.class).window(LastValueFunction.class).window(NthValueFunction.class).window(LagFunction.class).window(LeadFunction.class).aggregates(ApproximateCountDistinctAggregation.class).aggregates(DefaultApproximateCountDistinctAggregation.class).aggregates(BooleanApproximateCountDistinctAggregation.class).aggregates(BooleanDefaultApproximateCountDistinctAggregation.class).aggregates(SumDataSizeForStats.class).aggregates(MaxDataSizeForStats.class).aggregates(CountAggregation.class).aggregates(VarianceAggregation.class).aggregates(CentralMomentsAggregation.class).aggregates(ApproximateLongPercentileAggregations.class).aggregates(LegacyApproximateLongPercentileAggregations.class).aggregates(ApproximateLongPercentileArrayAggregations.class).aggregates(ApproximateDoublePercentileAggregations.class).aggregates(LegacyApproximateDoublePercentileAggregations.class).aggregates(ApproximateDoublePercentileArrayAggregations.class).aggregates(ApproximateRealPercentileAggregations.class).aggregates(LegacyApproximateRealPercentileAggregations.class).aggregates(ApproximateRealPercentileArrayAggregations.class).aggregates(CountIfAggregation.class).aggregates(BooleanAndAggregation.class).aggregates(BooleanOrAggregation.class).aggregates(DoubleSumAggregation.class).aggregates(RealSumAggregation.class).aggregates(LongSumAggregation.class).aggregates(IntervalDayToSecondSumAggregation.class).aggregates(IntervalYearToMonthSumAggregation.class).aggregates(AverageAggregations.class).function(REAL_AVERAGE_AGGREGATION).aggregates(IntervalDayToSecondAverageAggregation.class).aggregates(IntervalYearToMonthAverageAggregation.class).aggregates(GeometricMeanAggregations.class).aggregates(RealGeometricMeanAggregations.class).aggregates(MergeHyperLogLogAggregation.class).aggregates(ApproximateSetAggregation.class).aggregates(ApproximateSetGenericAggregation.class).aggregates(TDigestAggregationFunction.class).functions(QDIGEST_AGG, QDIGEST_AGG_WITH_WEIGHT, QDIGEST_AGG_WITH_WEIGHT_AND_ERROR).function(MergeQuantileDigestFunction.MERGE).aggregates(MergeTDigestAggregation.class).aggregates(DoubleHistogramAggregation.class).aggregates(RealHistogramAggregation.class).aggregates(DoubleCovarianceAggregation.class).aggregates(RealCovarianceAggregation.class).aggregates(DoubleRegressionAggregation.class).aggregates(RealRegressionAggregation.class).aggregates(DoubleCorrelationAggregation.class).aggregates(RealCorrelationAggregation.class).aggregates(BitwiseOrAggregation.class).aggregates(BitwiseAndAggregation.class).scalar(RepeatFunction.class).scalars(SequenceFunction.class).scalars(SessionFunctions.class).scalars(StringFunctions.class).scalars(WordStemFunction.class).scalar(SplitToMapFunction.class).scalar(SplitToMultimapFunction.class).scalars(VarbinaryFunctions.class).scalars(UrlFunctions.class).scalars(MathFunctions.class).scalar(MathFunctions.Abs.class).scalar(MathFunctions.Sign.class).scalar(MathFunctions.Round.class).scalar(MathFunctions.RoundN.class).scalar(MathFunctions.Truncate.class).scalar(MathFunctions.TruncateN.class).scalar(MathFunctions.Ceiling.class).scalar(MathFunctions.Floor.class).scalars(BitwiseFunctions.class).scalars(DateTimeFunctions.class).scalar(DateTimeFunctions.FromUnixtimeNanosDecimal.class).scalars(JsonFunctions.class).scalars(ColorFunctions.class).scalars(HyperLogLogFunctions.class).scalars(QuantileDigestFunctions.class).scalars(TDigestFunctions.class).scalars(BooleanOperators.class).scalars(BigintOperators.class).scalars(IntegerOperators.class).scalars(SmallintOperators.class).scalars(TinyintOperators.class).scalars(DoubleOperators.class).scalars(RealOperators.class).scalars(VarcharOperators.class).scalars(DateOperators.class).scalars(IntervalDayTimeOperators.class).scalars(IntervalYearMonthOperators.class).scalars(DateTimeOperators.class).scalars(HyperLogLogOperators.class).scalars(QuantileDigestOperators.class).scalars(TDigestOperators.class).scalars(IpAddressOperators.class).scalars(IpAddressFunctions.class).scalars(UuidOperators.class).scalars(LikeFunctions.class).scalars(ArrayFunctions.class).scalars(HmacFunctions.class).scalars(DataSizeFunctions.class).scalars(FormatNumberFunction.class).scalar(ArrayCardinalityFunction.class).scalar(ArrayContains.class).scalar(ArrayContainsSequence.class).scalar(ArrayFilterFunction.class).scalar(ArrayPositionFunction.class).scalars(CombineHashFunction.class).scalars(JsonOperators.class).scalars(FailureFunction.class).scalars(JoniRegexpCasts.class).scalars(CharacterStringCasts.class).scalars(LuhnCheckFunction.class).scalar(DecimalOperators.Negation.class).functions(IDENTITY_CAST, CAST_FROM_UNKNOWN).scalar(ArrayRemoveFunction.class).scalar(ArrayElementAtFunction.class).scalar(ArraySortFunction.class).scalar(ArraySortComparatorFunction.class).scalar(ArrayShuffleFunction.class).scalar(ArrayReverseFunction.class).scalar(ArrayMinFunction.class).scalar(ArrayMaxFunction.class).scalar(ArrayDistinctFunction.class).scalar(ArrayIntersectFunction.class).scalar(ArraysOverlapFunction.class).scalar(ArrayUnionFunction.class).scalar(ArrayExceptFunction.class).scalar(ArraySliceFunction.class).scalar(ArrayTrimFunction.class).scalar(ArrayCombinationsFunction.class).scalar(ArrayNgramsFunction.class).scalar(ArrayAllMatchFunction.class).scalar(ArrayAnyMatchFunction.class).scalar(ArrayNoneMatchFunction.class).scalar(MapEntriesFunction.class).scalar(MapFromEntriesFunction.class).scalar(MultimapFromEntriesFunction.class).scalar(MapKeys.class).scalar(MapValues.class).scalar(MapCardinalityFunction.class).scalar(EmptyMapConstructor.class).scalar(TypeOfFunction.class).scalar(TryFunction.class).scalar(ConcatWsFunction.ConcatArrayWs.class).scalar(DynamicFilters.Function.class).scalar(DynamicFilters.NullableFunction.class).functions(ZIP_WITH_FUNCTION, MAP_ZIP_WITH_FUNCTION).functions(ZIP_FUNCTIONS).functions(ARRAY_JOIN, ARRAY_JOIN_WITH_NULL_REPLACEMENT).scalar(ArrayToArrayCast.class).functions(ARRAY_TO_ELEMENT_CONCAT_FUNCTION, ELEMENT_TO_ARRAY_CONCAT_FUNCTION).function(MAP_ELEMENT_AT).function(new MapConcatFunction(blockTypeOperators)).function(new MapToMapCast(blockTypeOperators)).function(ARRAY_FLATTEN_FUNCTION).function(ARRAY_CONCAT_FUNCTION).functions(ARRAY_CONSTRUCTOR, ARRAY_SUBSCRIPT, JSON_TO_ARRAY, JSON_STRING_TO_ARRAY).function(ARRAY_AGG).function(LISTAGG).functions(new MapSubscriptOperator()).functions(MAP_CONSTRUCTOR, JSON_TO_MAP, JSON_STRING_TO_MAP).functions(new MapAggregationFunction(blockTypeOperators), new MapUnionAggregation(blockTypeOperators)).function(REDUCE_AGG).function(new MultimapAggregationFunction(blockTypeOperators)).functions(DECIMAL_TO_VARCHAR_CAST, DECIMAL_TO_INTEGER_CAST, DECIMAL_TO_BIGINT_CAST, DECIMAL_TO_DOUBLE_CAST, DECIMAL_TO_REAL_CAST, DECIMAL_TO_BOOLEAN_CAST, DECIMAL_TO_TINYINT_CAST, DECIMAL_TO_SMALLINT_CAST).functions(VARCHAR_TO_DECIMAL_CAST, INTEGER_TO_DECIMAL_CAST, BIGINT_TO_DECIMAL_CAST, DOUBLE_TO_DECIMAL_CAST, REAL_TO_DECIMAL_CAST, BOOLEAN_TO_DECIMAL_CAST, TINYINT_TO_DECIMAL_CAST, SMALLINT_TO_DECIMAL_CAST).functions(JSON_TO_DECIMAL_CAST, DECIMAL_TO_JSON_CAST).functions(DECIMAL_ADD_OPERATOR, DECIMAL_SUBTRACT_OPERATOR, DECIMAL_MULTIPLY_OPERATOR, DECIMAL_DIVIDE_OPERATOR, DECIMAL_MODULUS_OPERATOR).function(DECIMAL_TO_DECIMAL_SATURATED_FLOOR_CAST).functions(DECIMAL_TO_BIGINT_SATURATED_FLOOR_CAST, BIGINT_TO_DECIMAL_SATURATED_FLOOR_CAST).functions(DECIMAL_TO_INTEGER_SATURATED_FLOOR_CAST, INTEGER_TO_DECIMAL_SATURATED_FLOOR_CAST).functions(DECIMAL_TO_SMALLINT_SATURATED_FLOOR_CAST, SMALLINT_TO_DECIMAL_SATURATED_FLOOR_CAST).functions(DECIMAL_TO_TINYINT_SATURATED_FLOOR_CAST, TINYINT_TO_DECIMAL_SATURATED_FLOOR_CAST).function(new Histogram(blockTypeOperators)).function(new ChecksumAggregationFunction(blockTypeOperators)).function(ARBITRARY_AGGREGATION).functions(GREATEST, LEAST).functions(MAX_BY, MIN_BY, new MaxByNAggregationFunction(blockTypeOperators), new MinByNAggregationFunction(blockTypeOperators)).functions(MAX_AGGREGATION, MIN_AGGREGATION, new MaxNAggregationFunction(blockTypeOperators), new MinNAggregationFunction(blockTypeOperators)).function(COUNT_COLUMN).functions(JSON_TO_ROW, JSON_STRING_TO_ROW, ROW_TO_ROW_CAST).functions(VARCHAR_CONCAT, VARBINARY_CONCAT).function(CONCAT_WS).function(DECIMAL_TO_DECIMAL_CAST).function(castVarcharToRe2JRegexp(featuresConfig.getRe2JDfaStatesLimit(), featuresConfig.getRe2JDfaRetries())).function(castCharToRe2JRegexp(featuresConfig.getRe2JDfaStatesLimit(), featuresConfig.getRe2JDfaRetries())).function(DECIMAL_AVERAGE_AGGREGATION).function(DECIMAL_SUM_AGGREGATION).function(DECIMAL_MOD_FUNCTION).functions(ARRAY_TRANSFORM_FUNCTION, ARRAY_REDUCE_FUNCTION).functions(MAP_FILTER_FUNCTION, new MapTransformKeysFunction(blockTypeOperators), MAP_TRANSFORM_VALUES_FUNCTION).function(FORMAT_FUNCTION).function(TRY_CAST).function(new GenericEqualOperator(typeOperators)).function(new GenericHashCodeOperator(typeOperators)).function(new GenericXxHash64Operator(typeOperators)).function(new GenericDistinctFromOperator(typeOperators)).function(new GenericIndeterminateOperator(typeOperators)).function(new GenericComparisonUnorderedLastOperator(typeOperators)).function(new GenericComparisonUnorderedFirstOperator(typeOperators)).function(new GenericLessThanOperator(typeOperators)).function(new GenericLessThanOrEqualOperator(typeOperators)).function(new VersionFunction(nodeVersion.getVersion())).aggregates(MergeSetDigestAggregation.class).aggregates(BuildSetDigestAggregation.class).scalars(SetDigestFunctions.class).scalars(SetDigestOperators.class).scalars(WilsonInterval.class).aggregates(BigintApproximateMostFrequent.class).aggregates(VarcharApproximateMostFrequent.class);
    // timestamp operators and functions
    builder.scalar(TimestampOperators.TimestampPlusIntervalDayToSecond.class).scalar(TimestampOperators.IntervalDayToSecondPlusTimestamp.class).scalar(TimestampOperators.TimestampPlusIntervalYearToMonth.class).scalar(TimestampOperators.IntervalYearToMonthPlusTimestamp.class).scalar(TimestampOperators.TimestampMinusIntervalDayToSecond.class).scalar(TimestampOperators.TimestampMinusIntervalYearToMonth.class).scalar(TimestampOperators.TimestampMinusTimestamp.class).scalar(TimestampToTimestampCast.class).scalar(TimestampToTimeCast.class).scalar(TimestampToTimeWithTimeZoneCast.class).scalar(TimestampToTimestampWithTimeZoneCast.class).scalar(TimestampToDateCast.class).scalar(TimestampToVarcharCast.class).scalar(TimestampToJsonCast.class).scalar(DateToTimestampCast.class).scalar(TimeToTimestampCast.class).scalar(TimeWithTimeZoneToTimestampCast.class).scalar(TimestampWithTimeZoneToTimestampCast.class).scalar(VarcharToTimestampCast.class).scalar(LocalTimestamp.class).scalar(DateTrunc.class).scalar(HumanReadableSeconds.class).scalar(ToIso8601.class).scalar(WithTimeZone.class).scalar(FormatDateTime.class).scalar(DateFormat.class).scalar(SequenceIntervalYearToMonth.class).scalar(SequenceIntervalDayToSecond.class).scalar(DateAdd.class).scalar(DateDiff.class).scalar(ExtractYear.class).scalar(ExtractQuarter.class).scalar(ExtractMonth.class).scalar(ExtractDay.class).scalar(ExtractHour.class).scalar(ExtractMinute.class).scalar(ExtractSecond.class).scalar(ExtractMillisecond.class).scalar(ExtractDayOfYear.class).scalar(ExtractDayOfWeek.class).scalar(ExtractWeekOfYear.class).scalar(ExtractYearOfWeek.class).scalar(LastDayOfMonth.class);
    // timestamp with timezone operators and functions
    builder.scalar(TimestampWithTimeZoneOperators.TimestampPlusIntervalDayToSecond.class).scalar(TimestampWithTimeZoneOperators.IntervalDayToSecondPlusTimestamp.class).scalar(TimestampWithTimeZoneOperators.TimestampMinusIntervalDayToSecond.class).scalar(TimestampWithTimeZoneOperators.TimestampPlusIntervalYearToMonth.class).scalar(TimestampWithTimeZoneOperators.IntervalYearToMonthPlusTimestamp.class).scalar(TimestampWithTimeZoneOperators.TimestampMinusIntervalYearToMonth.class).scalar(TimestampWithTimeZoneOperators.TimestampMinusTimestamp.class).scalar(CurrentTimestamp.class).scalar(io.trino.operator.scalar.timestamptz.ExtractYear.class).scalar(io.trino.operator.scalar.timestamptz.ExtractQuarter.class).scalar(io.trino.operator.scalar.timestamptz.ExtractMonth.class).scalar(io.trino.operator.scalar.timestamptz.ExtractDay.class).scalar(io.trino.operator.scalar.timestamptz.ExtractHour.class).scalar(io.trino.operator.scalar.timestamptz.ExtractMinute.class).scalar(io.trino.operator.scalar.timestamptz.ExtractSecond.class).scalar(io.trino.operator.scalar.timestamptz.ExtractMillisecond.class).scalar(io.trino.operator.scalar.timestamptz.ExtractDayOfYear.class).scalar(io.trino.operator.scalar.timestamptz.ExtractDayOfWeek.class).scalar(io.trino.operator.scalar.timestamptz.ExtractWeekOfYear.class).scalar(io.trino.operator.scalar.timestamptz.ExtractYearOfWeek.class).scalar(io.trino.operator.scalar.timestamptz.ToIso8601.class).scalar(io.trino.operator.scalar.timestamptz.DateAdd.class).scalar(io.trino.operator.scalar.timestamptz.DateTrunc.class).scalar(io.trino.operator.scalar.timestamptz.TimeZoneHour.class).scalar(io.trino.operator.scalar.timestamptz.TimeZoneMinute.class).scalar(io.trino.operator.scalar.timestamptz.DateDiff.class).scalar(io.trino.operator.scalar.timestamptz.DateFormat.class).scalar(io.trino.operator.scalar.timestamptz.FormatDateTime.class).scalar(io.trino.operator.scalar.timestamptz.ToUnixTime.class).scalar(io.trino.operator.scalar.timestamptz.LastDayOfMonth.class).scalar(AtTimeZone.class).scalar(AtTimeZoneWithOffset.class).scalar(DateToTimestampWithTimeZoneCast.class).scalar(TimestampWithTimeZoneToDateCast.class).scalar(TimestampWithTimeZoneToTimeCast.class).scalar(TimestampWithTimeZoneToTimestampWithTimeZoneCast.class).scalar(TimestampWithTimeZoneToTimeWithTimeZoneCast.class).scalar(TimestampWithTimeZoneToVarcharCast.class).scalar(TimeToTimestampWithTimeZoneCast.class).scalar(TimeWithTimeZoneToTimestampWithTimeZoneCast.class).scalar(VarcharToTimestampWithTimeZoneCast.class);
    // time without time zone functions and operators
    builder.scalar(LocalTimeFunction.class).scalars(TimeOperators.class).scalars(TimeFunctions.class).scalar(TimeToTimeWithTimeZoneCast.class);
    // time with timezone operators and functions
    builder.scalar(TimeWithTimeZoneOperators.TimePlusIntervalDayToSecond.class).scalar(TimeWithTimeZoneOperators.IntervalDayToSecondPlusTime.class).scalar(TimeWithTimeZoneOperators.TimeMinusIntervalDayToSecond.class).scalar(TimeWithTimeZoneOperators.TimeMinusTime.class).scalar(TimeWithTimeZoneToTimeCast.class).scalar(TimeWithTimeZoneToTimeWithTimeZoneCast.class).scalar(TimeWithTimeZoneToVarcharCast.class).scalar(VarcharToTimeWithTimeZoneCast.class).scalar(io.trino.operator.scalar.timetz.DateDiff.class).scalar(io.trino.operator.scalar.timetz.DateAdd.class).scalar(io.trino.operator.scalar.timetz.ExtractHour.class).scalar(io.trino.operator.scalar.timetz.ExtractMinute.class).scalar(io.trino.operator.scalar.timetz.ExtractSecond.class).scalar(io.trino.operator.scalar.timetz.ExtractMillisecond.class).scalar(io.trino.operator.scalar.timetz.TimeZoneHour.class).scalar(io.trino.operator.scalar.timetz.TimeZoneMinute.class).scalar(io.trino.operator.scalar.timetz.DateTrunc.class).scalar(io.trino.operator.scalar.timetz.AtTimeZone.class).scalar(io.trino.operator.scalar.timetz.AtTimeZoneWithOffset.class).scalar(CurrentTime.class);
    switch(featuresConfig.getRegexLibrary()) {
        case JONI:
            builder.scalars(JoniRegexpFunctions.class);
            builder.scalar(JoniRegexpReplaceLambdaFunction.class);
            break;
        case RE2J:
            builder.scalars(Re2JRegexpFunctions.class);
            builder.scalar(Re2JRegexpReplaceLambdaFunction.class);
            break;
    }
    if (featuresConfig.isLegacyRowToJsonCast()) {
        builder.functions(LEGACY_ROW_TO_JSON, LEGACY_ARRAY_TO_JSON, LEGACY_MAP_TO_JSON);
    } else {
        builder.functions(ROW_TO_JSON, ARRAY_TO_JSON, MAP_TO_JSON);
    }
    return builder.build();
}