use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.
the class TypeExtractor method analyzePojo.
@SuppressWarnings("unchecked")
protected <OUT, IN1, IN2> TypeInformation<OUT> analyzePojo(Class<OUT> clazz, ArrayList<Type> typeHierarchy, ParameterizedType parameterizedType, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
if (!Modifier.isPublic(clazz.getModifiers())) {
LOG.info("Class " + clazz.getName() + " is not public, cannot treat it as a POJO type. Will be handled as GenericType");
return new GenericTypeInfo<OUT>(clazz);
}
// add the hierarchy of the POJO itself if it is generic
if (parameterizedType != null) {
getTypeHierarchy(typeHierarchy, parameterizedType, Object.class);
} else // create a type hierarchy, if the incoming only contains the most bottom one or none.
if (typeHierarchy.size() <= 1) {
getTypeHierarchy(typeHierarchy, clazz, Object.class);
}
List<Field> fields = getAllDeclaredFields(clazz, false);
if (fields.size() == 0) {
LOG.info("No fields detected for " + clazz + ". Cannot be used as a PojoType. Will be handled as GenericType");
return new GenericTypeInfo<OUT>(clazz);
}
List<PojoField> pojoFields = new ArrayList<PojoField>();
for (Field field : fields) {
Type fieldType = field.getGenericType();
if (!isValidPojoField(field, clazz, typeHierarchy)) {
LOG.info(clazz + " is not a valid POJO type");
return null;
}
try {
ArrayList<Type> fieldTypeHierarchy = new ArrayList<Type>(typeHierarchy);
fieldTypeHierarchy.add(fieldType);
TypeInformation<?> ti = createTypeInfoWithTypeHierarchy(fieldTypeHierarchy, fieldType, in1Type, in2Type);
pojoFields.add(new PojoField(field, ti));
} catch (InvalidTypesException e) {
Class<?> genericClass = Object.class;
if (isClassType(fieldType)) {
genericClass = typeToClass(fieldType);
}
pojoFields.add(new PojoField(field, new GenericTypeInfo<OUT>((Class<OUT>) genericClass)));
}
}
CompositeType<OUT> pojoType = new PojoTypeInfo<OUT>(clazz, pojoFields);
//
// Validate the correctness of the pojo.
// returning "null" will result create a generic type information.
//
List<Method> methods = getAllDeclaredMethods(clazz);
for (Method method : methods) {
if (method.getName().equals("readObject") || method.getName().equals("writeObject")) {
LOG.info(clazz + " contains custom serialization methods we do not call.");
return null;
}
}
// Try retrieving the default constructor, if it does not have one
// we cannot use this because the serializer uses it.
Constructor defaultConstructor = null;
try {
defaultConstructor = clazz.getDeclaredConstructor();
} catch (NoSuchMethodException e) {
if (clazz.isInterface() || Modifier.isAbstract(clazz.getModifiers())) {
LOG.info(clazz + " is abstract or an interface, having a concrete " + "type can increase performance.");
} else {
LOG.info(clazz + " must have a default constructor to be used as a POJO.");
return null;
}
}
if (defaultConstructor != null && !Modifier.isPublic(defaultConstructor.getModifiers())) {
LOG.info("The default constructor of " + clazz + " should be Public to be used as a POJO.");
return null;
}
// everything is checked, we return the pojo
return pojoType;
}
use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.
the class TypeExtractor method createSubTypesInfo.
/**
* Creates the TypeInformation for all elements of a type that expects a certain number of
* subtypes (e.g. TupleXX).
*
* @param originalType most concrete subclass
* @param definingType type that defines the number of subtypes (e.g. Tuple2 -> 2 subtypes)
* @param typeHierarchy necessary for type inference
* @param in1Type necessary for type inference
* @param in2Type necessary for type inference
* @param lenient decides whether exceptions should be thrown if a subtype can not be determined
* @return array containing TypeInformation of sub types or null if definingType contains
* more subtypes (fields) that defined
*/
private <IN1, IN2> TypeInformation<?>[] createSubTypesInfo(Type originalType, ParameterizedType definingType, ArrayList<Type> typeHierarchy, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, boolean lenient) {
Type[] subtypes = new Type[definingType.getActualTypeArguments().length];
// materialize possible type variables
for (int i = 0; i < subtypes.length; i++) {
final Type actualTypeArg = definingType.getActualTypeArguments()[i];
// materialize immediate TypeVariables
if (actualTypeArg instanceof TypeVariable<?>) {
subtypes[i] = materializeTypeVariable(typeHierarchy, (TypeVariable<?>) actualTypeArg);
} else // class or parameterized type
{
subtypes[i] = actualTypeArg;
}
}
TypeInformation<?>[] subTypesInfo = new TypeInformation<?>[subtypes.length];
for (int i = 0; i < subtypes.length; i++) {
final ArrayList<Type> subTypeHierarchy = new ArrayList<>(typeHierarchy);
subTypeHierarchy.add(subtypes[i]);
// try to derive the type info of the TypeVariable from the immediate base child input as a last attempt
if (subtypes[i] instanceof TypeVariable<?>) {
subTypesInfo[i] = createTypeInfoFromInputs((TypeVariable<?>) subtypes[i], subTypeHierarchy, in1Type, in2Type);
// variable could not be determined
if (subTypesInfo[i] == null && !lenient) {
throw new InvalidTypesException("Type of TypeVariable '" + ((TypeVariable<?>) subtypes[i]).getName() + "' in '" + ((TypeVariable<?>) subtypes[i]).getGenericDeclaration() + "' could not be determined. This is most likely a type erasure problem. " + "The type extraction currently supports types with generic variables only in cases where " + "all variables in the return type can be deduced from the input type(s).");
}
} else {
// create the type information of the subtype or null/exception
try {
subTypesInfo[i] = createTypeInfoWithTypeHierarchy(subTypeHierarchy, subtypes[i], in1Type, in2Type);
} catch (InvalidTypesException e) {
if (lenient) {
subTypesInfo[i] = null;
} else {
throw e;
}
}
}
}
// check that number of fields matches the number of subtypes
if (!lenient) {
Class<?> originalTypeAsClass = null;
if (isClassType(originalType)) {
originalTypeAsClass = typeToClass(originalType);
}
checkNotNull(originalTypeAsClass, "originalType has an unexpected type");
// check if the class we assumed to conform to the defining type so far is actually a pojo because the
// original type contains additional fields.
// check for additional fields.
int fieldCount = countFieldsInClass(originalTypeAsClass);
if (fieldCount > subTypesInfo.length) {
return null;
}
}
return subTypesInfo;
}
use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.
the class TypeExtractor method getUnaryOperatorReturnType.
/**
* Returns the unary operator's return type.
*
* @param function Function to extract the return type from
* @param baseClass Base class of the function
* @param inputTypeArgumentIndex Index of the type argument of function's first parameter
* specifying the input type if it is wrapped (Iterable, Map,
* etc.). Otherwise -1.
* @param outputTypeArgumentIndex Index of the type argument of functions second parameter
* specifying the output type if it is wrapped in a Collector.
* Otherwise -1.
* @param inType Type of the input elements (In case of an iterable, it is the element type)
* @param functionName Function name
* @param allowMissing Can the type information be missing
* @param <IN> Input type
* @param <OUT> Output type
* @return TypeInformation of the return type of the function
*/
@SuppressWarnings("unchecked")
@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getUnaryOperatorReturnType(Function function, Class<?> baseClass, int inputTypeArgumentIndex, int outputTypeArgumentIndex, TypeInformation<IN> inType, String functionName, boolean allowMissing) {
try {
final LambdaExecutable exec;
try {
exec = checkAndExtractLambda(function);
} catch (TypeExtractionException e) {
throw new InvalidTypesException("Internal error occurred.", e);
}
if (exec != null) {
// check for lambda type erasure
validateLambdaGenericParameters(exec);
// parameters must be accessed from behind, since JVM can add additional parameters e.g. when using local variables inside lambda function
final int paramLen = exec.getParameterTypes().length - 1;
// executable references "this" implicitly
if (paramLen < 0) {
// executable declaring class can also be a super class of the input type
// we only validate if the executable exists in input type
validateInputContainsExecutable(exec, inType);
} else {
final Type input = (outputTypeArgumentIndex >= 0) ? exec.getParameterTypes()[paramLen - 1] : exec.getParameterTypes()[paramLen];
validateInputType((inputTypeArgumentIndex >= 0) ? extractTypeArgument(input, inputTypeArgumentIndex) : input, inType);
}
if (function instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) function).getProducedType();
}
return new TypeExtractor().privateCreateTypeInfo((outputTypeArgumentIndex >= 0) ? extractTypeArgument(exec.getParameterTypes()[paramLen], outputTypeArgumentIndex) : exec.getReturnType(), inType, null);
} else {
validateInputType(baseClass, function.getClass(), 0, inType);
if (function instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) function).getProducedType();
}
return new TypeExtractor().privateCreateTypeInfo(baseClass, function.getClass(), 1, inType, null);
}
} catch (InvalidTypesException e) {
if (allowMissing) {
return (TypeInformation<OUT>) new MissingTypeInfo(functionName != null ? functionName : function.toString(), e);
} else {
throw e;
}
}
}
use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.
the class TypeExtractor method getParameterType.
private static Type getParameterType(Class<?> baseClass, ArrayList<Type> typeHierarchy, Class<?> clazz, int pos) {
if (typeHierarchy != null) {
typeHierarchy.add(clazz);
}
Type[] interfaceTypes = clazz.getGenericInterfaces();
// search in interfaces for base class
for (Type t : interfaceTypes) {
Type parameter = getParameterTypeFromGenericType(baseClass, typeHierarchy, t, pos);
if (parameter != null) {
return parameter;
}
}
// search in superclass for base class
Type t = clazz.getGenericSuperclass();
Type parameter = getParameterTypeFromGenericType(baseClass, typeHierarchy, t, pos);
if (parameter != null) {
return parameter;
}
throw new InvalidTypesException("The types of the interface " + baseClass.getName() + " could not be inferred. " + "Support for synthetic interfaces, lambdas, and generic or raw types is limited at this point");
}
use of org.apache.flink.api.common.functions.InvalidTypesException in project flink by apache.
the class StreamExecutionEnvironment method addSource.
/**
* Ads a data source with a custom type information thus opening a
* {@link DataStream}. Only in very special cases does the user need to
* support type information. Otherwise use
* {@link #addSource(org.apache.flink.streaming.api.functions.source.SourceFunction)}
*
* @param function
* the user defined function
* @param sourceName
* Name of the data source
* @param <OUT>
* type of the returned stream
* @param typeInfo
* the user defined type information for the stream
* @return the data stream constructed
*/
@SuppressWarnings("unchecked")
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {
if (typeInfo == null) {
if (function instanceof ResultTypeQueryable) {
typeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
} else {
try {
typeInfo = TypeExtractor.createTypeInfo(SourceFunction.class, function.getClass(), 0, null, null);
} catch (final InvalidTypesException e) {
typeInfo = (TypeInformation<OUT>) new MissingTypeInfo(sourceName, e);
}
}
}
boolean isParallel = function instanceof ParallelSourceFunction;
clean(function);
StreamSource<OUT, ?> sourceOperator;
if (function instanceof StoppableFunction) {
sourceOperator = new StoppableStreamSource<>(cast2StoppableSourceFunction(function));
} else {
sourceOperator = new StreamSource<>(function);
}
return new DataStreamSource<>(this, typeInfo, sourceOperator, isParallel, sourceName);
}
Aggregations