Examples with PythonFunctionInfo org.apache.flink.table.functions.python.PythonFunctionInfo used on opensource projects

Example 11 with PythonFunctionInfo

use of org.apache.flink.table.functions.python.PythonFunctionInfo in project flink by apache.

the class AbstractArrowPythonAggregateFunctionOperator method getUserDefinedFunctionsProto.

@Override
public FlinkFnApi.UserDefinedFunctions getUserDefinedFunctionsProto() {
    FlinkFnApi.UserDefinedFunctions.Builder builder = FlinkFnApi.UserDefinedFunctions.newBuilder();
    // add udaf proto
    for (PythonFunctionInfo pythonFunctionInfo : pandasAggFunctions) {
        builder.addUdfs(getUserDefinedFunctionProto(pythonFunctionInfo));
    }
    builder.setMetricEnabled(pythonConfig.isMetricEnabled());
    builder.setProfileEnabled(pythonConfig.isProfileEnabled());
    return builder.build();
}

Example 12 with PythonFunctionInfo

use of org.apache.flink.table.functions.python.PythonFunctionInfo in project flink by apache.

the class PythonTableFunctionOperatorTestBase method getTestHarness.

private OneInputStreamOperatorTestHarness<IN, OUT> getTestHarness(Configuration config, JoinRelType joinRelType) throws Exception {
    RowType inputType = new RowType(Arrays.asList(new RowType.RowField("f1", new VarCharType()), new RowType.RowField("f2", new VarCharType()), new RowType.RowField("f3", new BigIntType())));
    RowType outputType = new RowType(Arrays.asList(new RowType.RowField("f1", new VarCharType()), new RowType.RowField("f2", new VarCharType()), new RowType.RowField("f3", new BigIntType()), new RowType.RowField("f4", new BigIntType())));
    PythonTableFunctionOperator operator = getTestOperator(config, new PythonFunctionInfo(PythonScalarFunctionOperatorTestBase.DummyPythonFunction.INSTANCE, new Integer[] { 0 }), inputType, outputType, new int[] { 2 }, joinRelType);
    OneInputStreamOperatorTestHarness<IN, OUT> testHarness = new OneInputStreamOperatorTestHarness(operator);
    testHarness.getStreamConfig().setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.PYTHON, 0.5);
    return testHarness;
}

Example 13 with PythonFunctionInfo

use of org.apache.flink.table.functions.python.PythonFunctionInfo in project flink by apache.

the class PythonScalarFunctionOperatorTestBase method getTestHarness.

private OneInputStreamOperatorTestHarness<IN, OUT> getTestHarness(Configuration config) throws Exception {
    RowType dataType = new RowType(Arrays.asList(new RowType.RowField("f1", new VarCharType()), new RowType.RowField("f2", new VarCharType()), new RowType.RowField("f3", new BigIntType())));
    AbstractPythonScalarFunctionOperator operator = getTestOperator(config, new PythonFunctionInfo[] { new PythonFunctionInfo(DummyPythonFunction.INSTANCE, new Integer[] { 0 }) }, dataType, dataType, new int[] { 2 }, new int[] { 0, 1 });
    OneInputStreamOperatorTestHarness<IN, OUT> testHarness = new OneInputStreamOperatorTestHarness(operator);
    testHarness.getStreamConfig().setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.PYTHON, 0.5);
    testHarness.setup(getOutputTypeSerializer(dataType));
    return testHarness;
}

Example 14 with PythonFunctionInfo

use of org.apache.flink.table.functions.python.PythonFunctionInfo in project flink by apache.

the class CommonExecPythonCorrelate method createPythonOneInputTransformation.

private OneInputTransformation<RowData, RowData> createPythonOneInputTransformation(Transformation<RowData> inputTransform, ExecNodeConfig config, Configuration pythonConfig) {
    Tuple2<int[], PythonFunctionInfo> extractResult = extractPythonTableFunctionInfo();
    int[] pythonUdtfInputOffsets = extractResult.f0;
    PythonFunctionInfo pythonFunctionInfo = extractResult.f1;
    InternalTypeInfo<RowData> pythonOperatorInputRowType = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
    InternalTypeInfo<RowData> pythonOperatorOutputRowType = InternalTypeInfo.of((RowType) getOutputType());
    OneInputStreamOperator<RowData, RowData> pythonOperator = getPythonTableFunctionOperator(config, pythonConfig, pythonOperatorInputRowType, pythonOperatorOutputRowType, pythonFunctionInfo, pythonUdtfInputOffsets);
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationName(config), createTransformationDescription(config), pythonOperator, pythonOperatorOutputRowType, inputTransform.getParallelism());
}

Example 15 with PythonFunctionInfo

use of org.apache.flink.table.functions.python.PythonFunctionInfo in project flink by apache.

the class BatchExecPythonGroupWindowAggregate method getPythonGroupWindowAggregateFunctionOperator.

@SuppressWarnings("unchecked")
private OneInputStreamOperator<RowData, RowData> getPythonGroupWindowAggregateFunctionOperator(ExecNodeConfig config, Configuration pythonConfig, RowType inputRowType, RowType outputRowType, int maxLimitSize, long windowSize, long slideSize, int[] namePropertyTypeArray, int[] udafInputOffsets, PythonFunctionInfo[] pythonFunctionInfos) {
    Class<?> clazz = CommonPythonUtil.loadClass(ARROW_PYTHON_GROUP_WINDOW_AGGREGATE_FUNCTION_OPERATOR_NAME);
    RowType udfInputType = (RowType) Projection.of(udafInputOffsets).project(inputRowType);
    RowType udfOutputType = (RowType) Projection.range(auxGrouping.length, outputRowType.getFieldCount() - namePropertyTypeArray.length).project(outputRowType);
    try {
        Constructor<?> ctor = clazz.getConstructor(Configuration.class, PythonFunctionInfo[].class, RowType.class, RowType.class, RowType.class, int.class, int.class, long.class, long.class, int[].class, GeneratedProjection.class, GeneratedProjection.class, GeneratedProjection.class);
        return (OneInputStreamOperator<RowData, RowData>) ctor.newInstance(pythonConfig, pythonFunctionInfos, inputRowType, udfInputType, udfOutputType, inputTimeFieldIndex, maxLimitSize, windowSize, slideSize, namePropertyTypeArray, ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "UdafInputProjection", inputRowType, udfInputType, udafInputOffsets), ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "GroupKey", inputRowType, (RowType) Projection.of(grouping).project(inputRowType), grouping), ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "GroupSet", inputRowType, (RowType) Projection.of(auxGrouping).project(inputRowType), auxGrouping));
    } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new TableException("Python BatchArrowPythonGroupWindowAggregateFunctionOperator constructed failed.", e);
    }
}