Search in sources :

Example 1 with InvalidTypesException

use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.

the class TypeExtractor method analyzePojo.

@SuppressWarnings("unchecked")
protected <OUT, IN1, IN2> TypeInformation<OUT> analyzePojo(Class<OUT> clazz, ArrayList<Type> typeHierarchy, ParameterizedType parameterizedType, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
    if (!Modifier.isPublic(clazz.getModifiers())) {
        LOG.info("Class " + clazz.getName() + " is not public, cannot treat it as a POJO type. Will be handled as GenericType");
        return new GenericTypeInfo<OUT>(clazz);
    }
    // add the hierarchy of the POJO itself if it is generic
    if (parameterizedType != null) {
        getTypeHierarchy(typeHierarchy, parameterizedType, Object.class);
    } else // create a type hierarchy, if the incoming only contains the most bottom one or none.
    if (typeHierarchy.size() <= 1) {
        getTypeHierarchy(typeHierarchy, clazz, Object.class);
    }
    List<Field> fields = getAllDeclaredFields(clazz, false);
    if (fields.size() == 0) {
        LOG.info("No fields detected for " + clazz + ". Cannot be used as a PojoType. Will be handled as GenericType");
        return new GenericTypeInfo<OUT>(clazz);
    }
    List<PojoField> pojoFields = new ArrayList<PojoField>();
    for (Field field : fields) {
        Type fieldType = field.getGenericType();
        if (!isValidPojoField(field, clazz, typeHierarchy)) {
            LOG.info(clazz + " is not a valid POJO type");
            return null;
        }
        try {
            ArrayList<Type> fieldTypeHierarchy = new ArrayList<Type>(typeHierarchy);
            fieldTypeHierarchy.add(fieldType);
            TypeInformation<?> ti = createTypeInfoWithTypeHierarchy(fieldTypeHierarchy, fieldType, in1Type, in2Type);
            pojoFields.add(new PojoField(field, ti));
        } catch (InvalidTypesException e) {
            Class<?> genericClass = Object.class;
            if (isClassType(fieldType)) {
                genericClass = typeToClass(fieldType);
            }
            pojoFields.add(new PojoField(field, new GenericTypeInfo<OUT>((Class<OUT>) genericClass)));
        }
    }
    CompositeType<OUT> pojoType = new PojoTypeInfo<OUT>(clazz, pojoFields);
    //
    // Validate the correctness of the pojo.
    // returning "null" will result create a generic type information.
    //
    List<Method> methods = getAllDeclaredMethods(clazz);
    for (Method method : methods) {
        if (method.getName().equals("readObject") || method.getName().equals("writeObject")) {
            LOG.info(clazz + " contains custom serialization methods we do not call.");
            return null;
        }
    }
    // Try retrieving the default constructor, if it does not have one
    // we cannot use this because the serializer uses it.
    Constructor defaultConstructor = null;
    try {
        defaultConstructor = clazz.getDeclaredConstructor();
    } catch (NoSuchMethodException e) {
        if (clazz.isInterface() || Modifier.isAbstract(clazz.getModifiers())) {
            LOG.info(clazz + " is abstract or an interface, having a concrete " + "type can increase performance.");
        } else {
            LOG.info(clazz + " must have a default constructor to be used as a POJO.");
            return null;
        }
    }
    if (defaultConstructor != null && !Modifier.isPublic(defaultConstructor.getModifiers())) {
        LOG.info("The default constructor of " + clazz + " should be Public to be used as a POJO.");
        return null;
    }
    // everything is checked, we return the pojo
    return pojoType;
}
Also used : Constructor(java.lang.reflect.Constructor) ArrayList(java.util.ArrayList) Method(java.lang.reflect.Method) Field(java.lang.reflect.Field) GenericArrayType(java.lang.reflect.GenericArrayType) TypeExtractionUtils.isClassType(org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType) Type(java.lang.reflect.Type) CompositeType(org.apache.flink.api.common.typeutils.CompositeType) ParameterizedType(java.lang.reflect.ParameterizedType) TypeExtractionUtils.typeToClass(org.apache.flink.api.java.typeutils.TypeExtractionUtils.typeToClass) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException)

Example 2 with InvalidTypesException

use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.

the class TypeExtractor method createSubTypesInfo.

/**
	 * Creates the TypeInformation for all elements of a type that expects a certain number of
	 * subtypes (e.g. TupleXX).
	 *
	 * @param originalType most concrete subclass
	 * @param definingType type that defines the number of subtypes (e.g. Tuple2 -> 2 subtypes)
	 * @param typeHierarchy necessary for type inference
	 * @param in1Type necessary for type inference
	 * @param in2Type necessary for type inference
	 * @param lenient decides whether exceptions should be thrown if a subtype can not be determined
	 * @return array containing TypeInformation of sub types or null if definingType contains
	 *     more subtypes (fields) that defined
	 */
private <IN1, IN2> TypeInformation<?>[] createSubTypesInfo(Type originalType, ParameterizedType definingType, ArrayList<Type> typeHierarchy, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, boolean lenient) {
    Type[] subtypes = new Type[definingType.getActualTypeArguments().length];
    // materialize possible type variables
    for (int i = 0; i < subtypes.length; i++) {
        final Type actualTypeArg = definingType.getActualTypeArguments()[i];
        // materialize immediate TypeVariables
        if (actualTypeArg instanceof TypeVariable<?>) {
            subtypes[i] = materializeTypeVariable(typeHierarchy, (TypeVariable<?>) actualTypeArg);
        } else // class or parameterized type
        {
            subtypes[i] = actualTypeArg;
        }
    }
    TypeInformation<?>[] subTypesInfo = new TypeInformation<?>[subtypes.length];
    for (int i = 0; i < subtypes.length; i++) {
        final ArrayList<Type> subTypeHierarchy = new ArrayList<>(typeHierarchy);
        subTypeHierarchy.add(subtypes[i]);
        // try to derive the type info of the TypeVariable from the immediate base child input as a last attempt
        if (subtypes[i] instanceof TypeVariable<?>) {
            subTypesInfo[i] = createTypeInfoFromInputs((TypeVariable<?>) subtypes[i], subTypeHierarchy, in1Type, in2Type);
            // variable could not be determined
            if (subTypesInfo[i] == null && !lenient) {
                throw new InvalidTypesException("Type of TypeVariable '" + ((TypeVariable<?>) subtypes[i]).getName() + "' in '" + ((TypeVariable<?>) subtypes[i]).getGenericDeclaration() + "' could not be determined. This is most likely a type erasure problem. " + "The type extraction currently supports types with generic variables only in cases where " + "all variables in the return type can be deduced from the input type(s).");
            }
        } else {
            // create the type information of the subtype or null/exception
            try {
                subTypesInfo[i] = createTypeInfoWithTypeHierarchy(subTypeHierarchy, subtypes[i], in1Type, in2Type);
            } catch (InvalidTypesException e) {
                if (lenient) {
                    subTypesInfo[i] = null;
                } else {
                    throw e;
                }
            }
        }
    }
    // check that number of fields matches the number of subtypes
    if (!lenient) {
        Class<?> originalTypeAsClass = null;
        if (isClassType(originalType)) {
            originalTypeAsClass = typeToClass(originalType);
        }
        checkNotNull(originalTypeAsClass, "originalType has an unexpected type");
        // check if the class we assumed to conform to the defining type so far is actually a pojo because the
        // original type contains additional fields.
        // check for additional fields.
        int fieldCount = countFieldsInClass(originalTypeAsClass);
        if (fieldCount > subTypesInfo.length) {
            return null;
        }
    }
    return subTypesInfo;
}
Also used : GenericArrayType(java.lang.reflect.GenericArrayType) TypeExtractionUtils.isClassType(org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType) Type(java.lang.reflect.Type) CompositeType(org.apache.flink.api.common.typeutils.CompositeType) ParameterizedType(java.lang.reflect.ParameterizedType) TypeVariable(java.lang.reflect.TypeVariable) ArrayList(java.util.ArrayList) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation)

Example 3 with InvalidTypesException

use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.

the class TypeExtractor method getUnaryOperatorReturnType.

/**
	 * Returns the unary operator's return type.
	 *
	 * @param function Function to extract the return type from
	 * @param baseClass Base class of the function
	 * @param inputTypeArgumentIndex Index of the type argument of function's first parameter
	 *                               specifying the input type if it is wrapped (Iterable, Map,
	 *                               etc.). Otherwise -1.
	 * @param outputTypeArgumentIndex Index of the type argument of functions second parameter
	 *                                specifying the output type if it is wrapped in a Collector.
	 *                                Otherwise -1.
	 * @param inType Type of the input elements (In case of an iterable, it is the element type)
	 * @param functionName Function name
	 * @param allowMissing Can the type information be missing
	 * @param <IN> Input type
	 * @param <OUT> Output type
	 * @return TypeInformation of the return type of the function
	 */
@SuppressWarnings("unchecked")
@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getUnaryOperatorReturnType(Function function, Class<?> baseClass, int inputTypeArgumentIndex, int outputTypeArgumentIndex, TypeInformation<IN> inType, String functionName, boolean allowMissing) {
    try {
        final LambdaExecutable exec;
        try {
            exec = checkAndExtractLambda(function);
        } catch (TypeExtractionException e) {
            throw new InvalidTypesException("Internal error occurred.", e);
        }
        if (exec != null) {
            // check for lambda type erasure
            validateLambdaGenericParameters(exec);
            // parameters must be accessed from behind, since JVM can add additional parameters e.g. when using local variables inside lambda function
            final int paramLen = exec.getParameterTypes().length - 1;
            // executable references "this" implicitly
            if (paramLen < 0) {
                // executable declaring class can also be a super class of the input type
                // we only validate if the executable exists in input type
                validateInputContainsExecutable(exec, inType);
            } else {
                final Type input = (outputTypeArgumentIndex >= 0) ? exec.getParameterTypes()[paramLen - 1] : exec.getParameterTypes()[paramLen];
                validateInputType((inputTypeArgumentIndex >= 0) ? extractTypeArgument(input, inputTypeArgumentIndex) : input, inType);
            }
            if (function instanceof ResultTypeQueryable) {
                return ((ResultTypeQueryable<OUT>) function).getProducedType();
            }
            return new TypeExtractor().privateCreateTypeInfo((outputTypeArgumentIndex >= 0) ? extractTypeArgument(exec.getParameterTypes()[paramLen], outputTypeArgumentIndex) : exec.getReturnType(), inType, null);
        } else {
            validateInputType(baseClass, function.getClass(), 0, inType);
            if (function instanceof ResultTypeQueryable) {
                return ((ResultTypeQueryable<OUT>) function).getProducedType();
            }
            return new TypeExtractor().privateCreateTypeInfo(baseClass, function.getClass(), 1, inType, null);
        }
    } catch (InvalidTypesException e) {
        if (allowMissing) {
            return (TypeInformation<OUT>) new MissingTypeInfo(functionName != null ? functionName : function.toString(), e);
        } else {
            throw e;
        }
    }
}
Also used : GenericArrayType(java.lang.reflect.GenericArrayType) TypeExtractionUtils.isClassType(org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType) Type(java.lang.reflect.Type) CompositeType(org.apache.flink.api.common.typeutils.CompositeType) ParameterizedType(java.lang.reflect.ParameterizedType) LambdaExecutable(org.apache.flink.api.java.typeutils.TypeExtractionUtils.LambdaExecutable) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException) PublicEvolving(org.apache.flink.annotation.PublicEvolving)

Example 4 with InvalidTypesException

use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.

the class TypeExtractor method getParameterType.

private static Type getParameterType(Class<?> baseClass, ArrayList<Type> typeHierarchy, Class<?> clazz, int pos) {
    if (typeHierarchy != null) {
        typeHierarchy.add(clazz);
    }
    Type[] interfaceTypes = clazz.getGenericInterfaces();
    // search in interfaces for base class
    for (Type t : interfaceTypes) {
        Type parameter = getParameterTypeFromGenericType(baseClass, typeHierarchy, t, pos);
        if (parameter != null) {
            return parameter;
        }
    }
    // search in superclass for base class
    Type t = clazz.getGenericSuperclass();
    Type parameter = getParameterTypeFromGenericType(baseClass, typeHierarchy, t, pos);
    if (parameter != null) {
        return parameter;
    }
    throw new InvalidTypesException("The types of the interface " + baseClass.getName() + " could not be inferred. " + "Support for synthetic interfaces, lambdas, and generic or raw types is limited at this point");
}
Also used : GenericArrayType(java.lang.reflect.GenericArrayType) TypeExtractionUtils.isClassType(org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType) Type(java.lang.reflect.Type) CompositeType(org.apache.flink.api.common.typeutils.CompositeType) ParameterizedType(java.lang.reflect.ParameterizedType) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException)

Example 5 with InvalidTypesException

use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.

the class StreamExecutionEnvironment method addSource.

/**
	 * Ads a data source with a custom type information thus opening a
	 * {@link DataStream}. Only in very special cases does the user need to
	 * support type information. Otherwise use
	 * {@link #addSource(org.apache.flink.streaming.api.functions.source.SourceFunction)}
	 *
	 * @param function
	 * 		the user defined function
	 * @param sourceName
	 * 		Name of the data source
	 * @param <OUT>
	 * 		type of the returned stream
	 * @param typeInfo
	 * 		the user defined type information for the stream
	 * @return the data stream constructed
	 */
@SuppressWarnings("unchecked")
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {
    if (typeInfo == null) {
        if (function instanceof ResultTypeQueryable) {
            typeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
        } else {
            try {
                typeInfo = TypeExtractor.createTypeInfo(SourceFunction.class, function.getClass(), 0, null, null);
            } catch (final InvalidTypesException e) {
                typeInfo = (TypeInformation<OUT>) new MissingTypeInfo(sourceName, e);
            }
        }
    }
    boolean isParallel = function instanceof ParallelSourceFunction;
    clean(function);
    StreamSource<OUT, ?> sourceOperator;
    if (function instanceof StoppableFunction) {
        sourceOperator = new StoppableStreamSource<>(cast2StoppableSourceFunction(function));
    } else {
        sourceOperator = new StreamSource<>(function);
    }
    return new DataStreamSource<>(this, typeInfo, sourceOperator, isParallel, sourceName);
}
Also used : ParallelSourceFunction(org.apache.flink.streaming.api.functions.source.ParallelSourceFunction) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) InputFormatSourceFunction(org.apache.flink.streaming.api.functions.source.InputFormatSourceFunction) MissingTypeInfo(org.apache.flink.api.java.typeutils.MissingTypeInfo) ResultTypeQueryable(org.apache.flink.api.java.typeutils.ResultTypeQueryable) DataStreamSource(org.apache.flink.streaming.api.datastream.DataStreamSource) StoppableFunction(org.apache.flink.api.common.functions.StoppableFunction) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) ParallelSourceFunction(org.apache.flink.streaming.api.functions.source.ParallelSourceFunction) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException)

Aggregations

InvalidTypesException (org.apache.flink.api.common.functions.InvalidTypesException)31 GenericArrayType (java.lang.reflect.GenericArrayType)19 ParameterizedType (java.lang.reflect.ParameterizedType)19 Type (java.lang.reflect.Type)19 CompositeType (org.apache.flink.api.common.typeutils.CompositeType)19 TypeExtractionUtils.isClassType (org.apache.flink.api.java.typeutils.TypeExtractionUtils.isClassType)19 ArrayList (java.util.ArrayList)12 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)12 Field (java.lang.reflect.Field)6 TypeVariable (java.lang.reflect.TypeVariable)6 PrimitiveArrayTypeInfo (org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo)6 Method (java.lang.reflect.Method)5 PublicEvolving (org.apache.flink.annotation.PublicEvolving)5 BasicArrayTypeInfo (org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo)5 Tuple (org.apache.flink.api.java.tuple.Tuple)5 TypeExtractionUtils.typeToClass (org.apache.flink.api.java.typeutils.TypeExtractionUtils.typeToClass)5 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)4 LambdaExecutable (org.apache.flink.api.java.typeutils.TypeExtractionUtils.LambdaExecutable)4 SqlTimeTypeInfo (org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo)3 Constructor (java.lang.reflect.Constructor)2