Examples with InternalAggregationFunction com.facebook.presto.operator.aggregation.InternalAggregationFunction used on opensource projects

Example 11 with InternalAggregationFunction

use of com.facebook.presto.operator.aggregation.InternalAggregationFunction in project presto by prestodb.

the class TestHashAggregationOperator method testMemoryLimit.

@Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of 10B.*")
public void testMemoryLimit(boolean hashEnabled) {
    InternalAggregationFunction maxVarcharColumn = getAggregation("max", VARCHAR);
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR, BIGINT, VARCHAR, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(10, Unit.BYTE)).addPipelineContext(0, true, true, false).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(new DataSize(16, MEGABYTE)), joinCompiler, false);
    toPages(operatorFactory, driverContext, input);
}

Example 12 with InternalAggregationFunction

use of com.facebook.presto.operator.aggregation.InternalAggregationFunction in project presto by prestodb.

the class AbstractMinMaxByNAggregationFunction method generateAggregation.

protected InternalAggregationFunction generateAggregation(Type valueType, Type keyType) {
    DynamicClassLoader classLoader = new DynamicClassLoader(AbstractMinMaxNAggregationFunction.class.getClassLoader());
    BlockComparator comparator = typeToComparator.apply(keyType);
    List<Type> inputTypes = ImmutableList.of(valueType, keyType, BIGINT);
    MinMaxByNStateSerializer stateSerializer = new MinMaxByNStateSerializer(comparator, keyType, valueType);
    Type intermediateType = stateSerializer.getSerializedType();
    ArrayType outputType = new ArrayType(valueType);
    List<AggregationMetadata.ParameterMetadata> inputParameterMetadata = ImmutableList.of(new AggregationMetadata.ParameterMetadata(STATE), new AggregationMetadata.ParameterMetadata(NULLABLE_BLOCK_INPUT_CHANNEL, valueType), new AggregationMetadata.ParameterMetadata(BLOCK_INPUT_CHANNEL, keyType), new AggregationMetadata.ParameterMetadata(BLOCK_INDEX), new AggregationMetadata.ParameterMetadata(INPUT_CHANNEL, BIGINT));
    AggregationMetadata metadata = new AggregationMetadata(generateAggregationName(name, valueType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), inputParameterMetadata, INPUT_FUNCTION.bindTo(comparator).bindTo(valueType).bindTo(keyType), COMBINE_FUNCTION, OUTPUT_FUNCTION.bindTo(outputType), ImmutableList.of(new AccumulatorStateDescriptor(MinMaxByNState.class, stateSerializer, new MinMaxByNStateFactory())), outputType);
    GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader);
    return new InternalAggregationFunction(name, inputTypes, ImmutableList.of(intermediateType), outputType, true, false, factory);
}

Example 13 with InternalAggregationFunction

use of com.facebook.presto.operator.aggregation.InternalAggregationFunction in project presto by prestodb.

the class AbstractMinMaxBy method generateAggregation.

private InternalAggregationFunction generateAggregation(Type valueType, Type keyType, FunctionAndTypeManager functionAndTypeManager) {
    Class<?> stateClazz = getStateClass(keyType.getJavaType(), valueType.getJavaType());
    DynamicClassLoader classLoader = new DynamicClassLoader(getClass().getClassLoader());
    // Generate states and serializers:
    // For value that is a Block or Slice, we store them as Block/position combination
    // to avoid generating long-living objects through getSlice or getBlock.
    // This can also help reducing cross-region reference in G1GC engine.
    // TODO: keys can have the same problem. But usually they are primitive types (given the nature of comparison).
    AccumulatorStateFactory<?> stateFactory;
    AccumulatorStateSerializer<?> stateSerializer;
    if (valueType.getJavaType().isPrimitive()) {
        Map<String, Type> stateFieldTypes = ImmutableMap.of("First", keyType, "Second", valueType);
        stateFactory = StateCompiler.generateStateFactory(stateClazz, stateFieldTypes, classLoader);
        stateSerializer = StateCompiler.generateStateSerializer(stateClazz, stateFieldTypes, classLoader);
    } else {
        // StateCompiler checks type compatibility.
        // Given "Second" in this case is always a Block, we only need to make sure the getter and setter of the Blocks are properly generated.
        // We deliberately make "SecondBlock" an array type so that the compiler will treat it as a block to workaround the sanity check.
        stateFactory = StateCompiler.generateStateFactory(stateClazz, ImmutableMap.of("First", keyType, "SecondBlock", new ArrayType(valueType)), classLoader);
        // States can be generated by StateCompiler given the they are simply classes with getters and setters.
        // However, serializers have logic in it. Creating serializers is better than generating them.
        stateSerializer = getStateSerializer(keyType, valueType);
    }
    Type intermediateType = stateSerializer.getSerializedType();
    List<Type> inputTypes = ImmutableList.of(valueType, keyType);
    CallSiteBinder binder = new CallSiteBinder();
    OperatorType operator = min ? LESS_THAN : GREATER_THAN;
    MethodHandle compareMethod = functionAndTypeManager.getJavaScalarFunctionImplementation(functionAndTypeManager.resolveOperator(operator, fromTypes(keyType, keyType))).getMethodHandle();
    ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName("processMaxOrMinBy"), type(Object.class));
    definition.declareDefaultConstructor(a(PRIVATE));
    generateInputMethod(definition, binder, compareMethod, keyType, valueType, stateClazz);
    generateCombineMethod(definition, binder, compareMethod, keyType, valueType, stateClazz);
    generateOutputMethod(definition, binder, valueType, stateClazz);
    Class<?> generatedClass = defineClass(definition, Object.class, binder.getBindings(), classLoader);
    MethodHandle inputMethod = methodHandle(generatedClass, "input", stateClazz, Block.class, Block.class, int.class);
    MethodHandle combineMethod = methodHandle(generatedClass, "combine", stateClazz, stateClazz);
    MethodHandle outputMethod = methodHandle(generatedClass, "output", stateClazz, BlockBuilder.class);
    AggregationMetadata metadata = new AggregationMetadata(generateAggregationName(getSignature().getNameSuffix(), valueType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), createInputParameterMetadata(valueType, keyType), inputMethod, combineMethod, outputMethod, ImmutableList.of(new AccumulatorStateDescriptor(stateClazz, stateSerializer, stateFactory)), valueType);
    GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader);
    return new InternalAggregationFunction(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), valueType, true, false, factory);
}

Example 14 with InternalAggregationFunction

use of com.facebook.presto.operator.aggregation.InternalAggregationFunction in project presto by prestodb.

the class ArrayAggregationFunction method generateAggregation.

private static InternalAggregationFunction generateAggregation(Type type, boolean legacyArrayAgg, ArrayAggGroupImplementation groupMode) {
    DynamicClassLoader classLoader = new DynamicClassLoader(ArrayAggregationFunction.class.getClassLoader());
    AccumulatorStateSerializer<?> stateSerializer = new ArrayAggregationStateSerializer(type);
    AccumulatorStateFactory<?> stateFactory = new ArrayAggregationStateFactory(type, groupMode);
    List<Type> inputTypes = ImmutableList.of(type);
    Type outputType = new ArrayType(type);
    Type intermediateType = stateSerializer.getSerializedType();
    List<ParameterMetadata> inputParameterMetadata = createInputParameterMetadata(type, legacyArrayAgg);
    MethodHandle inputFunction = INPUT_FUNCTION.bindTo(type);
    MethodHandle combineFunction = COMBINE_FUNCTION.bindTo(type);
    MethodHandle outputFunction = OUTPUT_FUNCTION.bindTo(type);
    Class<? extends AccumulatorState> stateInterface = ArrayAggregationState.class;
    AggregationMetadata metadata = new AggregationMetadata(generateAggregationName(NAME, type.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), inputParameterMetadata, inputFunction, combineFunction, outputFunction, ImmutableList.of(new AccumulatorStateDescriptor(stateInterface, stateSerializer, stateFactory)), outputType);
    GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader);
    return new InternalAggregationFunction(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, true, true, factory);
}

Example 15 with InternalAggregationFunction

use of com.facebook.presto.operator.aggregation.InternalAggregationFunction in project presto by prestodb.

the class SetUnionFunction method specialize.

@Override
public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, FunctionAndTypeManager functionAndTypeManager) {
    Type elementType = boundVariables.getTypeVariable("T");
    ArrayType arrayType = (ArrayType) functionAndTypeManager.getParameterizedType(StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType.getTypeSignature())));
    DynamicClassLoader classLoader = new DynamicClassLoader(SetUnionFunction.class.getClassLoader());
    AccumulatorStateSerializer<?> stateSerializer = new SetAggregationStateSerializer(arrayType);
    AccumulatorStateFactory<?> stateFactory = new SetAggregationStateFactory(elementType);
    Class<? extends AccumulatorState> stateInterface = SetAggregationState.class;
    List<ParameterMetadata> inputParameterMetadata = ImmutableList.of(new ParameterMetadata(STATE), new ParameterMetadata(NULLABLE_BLOCK_INPUT_CHANNEL, elementType), new ParameterMetadata(BLOCK_INDEX));
    AggregationMetadata metadata = new AggregationMetadata(AggregationUtils.generateAggregationName(NAME, arrayType.getTypeSignature(), ImmutableList.of(elementType.getTypeSignature())), inputParameterMetadata, INPUT_FUNCTION.bindTo(elementType).bindTo(arrayType), COMBINE_FUNCTION, OUTPUT_FUNCTION, ImmutableList.of(new AccumulatorStateDescriptor(stateInterface, stateSerializer, stateFactory)), arrayType);
    GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader);
    return new InternalAggregationFunction(NAME, ImmutableList.of(elementType), ImmutableList.of(stateSerializer.getSerializedType()), arrayType, true, true, factory);
}