Search in sources :

Example 1 with MapViewSpec

use of org.apache.flink.table.runtime.dataview.MapViewSpec in project flink by apache.

the class DataViewUtils method extractDataViews.

/**
 * Searches for data views in the data type of an accumulator and extracts them.
 */
public static List<DataViewSpec> extractDataViews(int aggIndex, DataType accumulatorDataType) {
    final LogicalType accumulatorType = accumulatorDataType.getLogicalType();
    if (!accumulatorType.is(ROW) && !accumulatorType.is(STRUCTURED_TYPE)) {
        return Collections.emptyList();
    }
    final List<String> fieldNames = getFieldNames(accumulatorType);
    final List<DataType> fieldDataTypes = accumulatorDataType.getChildren();
    final List<DataViewSpec> specs = new ArrayList<>();
    for (int fieldIndex = 0; fieldIndex < fieldDataTypes.size(); fieldIndex++) {
        final DataType fieldDataType = fieldDataTypes.get(fieldIndex);
        final LogicalType fieldType = fieldDataType.getLogicalType();
        if (isDataView(fieldType, ListView.class)) {
            specs.add(new ListViewSpec(createStateId(aggIndex, fieldNames.get(fieldIndex)), fieldIndex, fieldDataType.getChildren().get(0)));
        } else if (isDataView(fieldType, MapView.class)) {
            specs.add(new MapViewSpec(createStateId(aggIndex, fieldNames.get(fieldIndex)), fieldIndex, fieldDataType.getChildren().get(0), false));
        }
        if (fieldType.getChildren().stream().anyMatch(c -> hasNested(c, t -> isDataView(t, DataView.class)))) {
            throw new TableException("Data views are only supported in the first level of a composite accumulator type.");
        }
    }
    return specs;
}
Also used : MapView(org.apache.flink.table.api.dataview.MapView) DataType(org.apache.flink.table.types.DataType) DataViewSpec(org.apache.flink.table.runtime.dataview.DataViewSpec) StructuredType(org.apache.flink.table.types.logical.StructuredType) ListViewSpec(org.apache.flink.table.runtime.dataview.ListViewSpec) Function(java.util.function.Function) ArrayList(java.util.ArrayList) RawType(org.apache.flink.table.types.logical.RawType) DataView(org.apache.flink.table.api.dataview.DataView) LogicalTypeChecks.hasNested(org.apache.flink.table.types.logical.utils.LogicalTypeChecks.hasNested) ListView(org.apache.flink.table.api.dataview.ListView) LazyBinaryFormat(org.apache.flink.table.data.binary.LazyBinaryFormat) ROW(org.apache.flink.table.types.logical.LogicalTypeRoot.ROW) DataTypeUtils(org.apache.flink.table.types.utils.DataTypeUtils) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) NullSerializer(org.apache.flink.table.dataview.NullSerializer) LogicalTypeChecks.getFieldNames(org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldNames) TableException(org.apache.flink.table.api.TableException) MapViewSpec(org.apache.flink.table.runtime.dataview.MapViewSpec) DataTypes(org.apache.flink.table.api.DataTypes) TypeTransformation(org.apache.flink.table.types.inference.TypeTransformation) List(java.util.List) STRUCTURED_TYPE(org.apache.flink.table.types.logical.LogicalTypeRoot.STRUCTURED_TYPE) LogicalType(org.apache.flink.table.types.logical.LogicalType) Internal(org.apache.flink.annotation.Internal) ExternalSerializer(org.apache.flink.table.runtime.typeutils.ExternalSerializer) Collections(java.util.Collections) TableException(org.apache.flink.table.api.TableException) DataViewSpec(org.apache.flink.table.runtime.dataview.DataViewSpec) ArrayList(java.util.ArrayList) LogicalType(org.apache.flink.table.types.logical.LogicalType) MapViewSpec(org.apache.flink.table.runtime.dataview.MapViewSpec) ListViewSpec(org.apache.flink.table.runtime.dataview.ListViewSpec) DataType(org.apache.flink.table.types.DataType) MapView(org.apache.flink.table.api.dataview.MapView)

Example 2 with MapViewSpec

use of org.apache.flink.table.runtime.dataview.MapViewSpec in project flink by apache.

the class CommonPythonUtil method extractDataViewSpecs.

public static DataViewSpec[] extractDataViewSpecs(int index, DataType accType) {
    if (!(accType instanceof FieldsDataType)) {
        return new DataViewSpec[0];
    }
    FieldsDataType compositeAccType = (FieldsDataType) accType;
    if (includesDataView(compositeAccType)) {
        LogicalType logicalType = compositeAccType.getLogicalType();
        if (logicalType instanceof RowType) {
            List<DataType> childrenDataTypes = compositeAccType.getChildren();
            return IntStream.range(0, childrenDataTypes.size()).mapToObj(i -> {
                DataType childDataType = childrenDataTypes.get(i);
                LogicalType childLogicalType = childDataType.getLogicalType();
                if ((childLogicalType instanceof RowType) && includesDataView((FieldsDataType) childDataType)) {
                    throw new TableException("For Python AggregateFunction, DataView cannot be used in the" + " nested columns of the accumulator. ");
                } else if ((childLogicalType instanceof StructuredType) && ListView.class.isAssignableFrom(((StructuredType) childLogicalType).getImplementationClass().get())) {
                    return new ListViewSpec("agg" + index + "$" + ((RowType) logicalType).getFieldNames().get(i), i, childDataType.getChildren().get(0));
                } else if ((childLogicalType instanceof StructuredType) && MapView.class.isAssignableFrom(((StructuredType) childLogicalType).getImplementationClass().get())) {
                    return new MapViewSpec("agg" + index + "$" + ((RowType) logicalType).getFieldNames().get(i), i, childDataType.getChildren().get(0), false);
                }
                return null;
            }).filter(Objects::nonNull).toArray(DataViewSpec[]::new);
        } else {
            throw new TableException("For Python AggregateFunction you can only use DataView in " + "Row type.");
        }
    } else {
        return new DataViewSpec[0];
    }
}
Also used : FieldsDataType(org.apache.flink.table.types.FieldsDataType) TableException(org.apache.flink.table.api.TableException) MapViewSpec(org.apache.flink.table.runtime.dataview.MapViewSpec) DataViewSpec(org.apache.flink.table.runtime.dataview.DataViewSpec) ListViewSpec(org.apache.flink.table.runtime.dataview.ListViewSpec) LogicalType(org.apache.flink.table.types.logical.LogicalType) RowType(org.apache.flink.table.types.logical.RowType) DataType(org.apache.flink.table.types.DataType) FieldsDataType(org.apache.flink.table.types.FieldsDataType) MapView(org.apache.flink.table.api.dataview.MapView) StructuredType(org.apache.flink.table.types.logical.StructuredType)

Aggregations

TableException (org.apache.flink.table.api.TableException)2 MapView (org.apache.flink.table.api.dataview.MapView)2 DataViewSpec (org.apache.flink.table.runtime.dataview.DataViewSpec)2 ListViewSpec (org.apache.flink.table.runtime.dataview.ListViewSpec)2 MapViewSpec (org.apache.flink.table.runtime.dataview.MapViewSpec)2 DataType (org.apache.flink.table.types.DataType)2 LogicalType (org.apache.flink.table.types.logical.LogicalType)2 StructuredType (org.apache.flink.table.types.logical.StructuredType)2 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 List (java.util.List)1 Function (java.util.function.Function)1 Internal (org.apache.flink.annotation.Internal)1 TypeSerializer (org.apache.flink.api.common.typeutils.TypeSerializer)1 DataTypes (org.apache.flink.table.api.DataTypes)1 DataView (org.apache.flink.table.api.dataview.DataView)1 ListView (org.apache.flink.table.api.dataview.ListView)1 LazyBinaryFormat (org.apache.flink.table.data.binary.LazyBinaryFormat)1 NullSerializer (org.apache.flink.table.dataview.NullSerializer)1 ExternalSerializer (org.apache.flink.table.runtime.typeutils.ExternalSerializer)1