Examples with TupleTypeInfo org.apache.flink.api.java.typeutils.TupleTypeInfo used on opensource projects

Example 31 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class PojoSerializerTest method testTuplePojoTestEquality.

/**
	 * This tests if the hashes returned by the pojo and tuple comparators are the same
	 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTuplePojoTestEquality() {
    // test with a simple, string-key first.
    PojoTypeInfo<TestUserClass> pType = (PojoTypeInfo<TestUserClass>) type;
    List<FlatFieldDescriptor> result = new ArrayList<FlatFieldDescriptor>();
    pType.getFlatFields("nestedClass.dumm2", 0, result);
    // see below
    int[] fields = new int[1];
    fields[0] = result.get(0).getPosition();
    TypeComparator<TestUserClass> pojoComp = pType.createComparator(fields, new boolean[] { true }, 0, new ExecutionConfig());
    TestUserClass pojoTestRecord = new TestUserClass(0, "abc", 3d, new int[] { 1, 2, 3 }, new Date(), new NestedTestUserClass(1, "haha", 4d, new int[] { 5, 4, 3 }));
    int pHash = pojoComp.hash(pojoTestRecord);
    Tuple1<String> tupleTest = new Tuple1<String>("haha");
    TupleTypeInfo<Tuple1<String>> tType = (TupleTypeInfo<Tuple1<String>>) TypeExtractor.getForObject(tupleTest);
    TypeComparator<Tuple1<String>> tupleComp = tType.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
    int tHash = tupleComp.hash(tupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable", pHash == tHash);
    // its important here to use the same values.
    Tuple3<Integer, String, Double> multiTupleTest = new Tuple3<Integer, String, Double>(1, "haha", 4d);
    TupleTypeInfo<Tuple3<Integer, String, Double>> multiTupleType = (TupleTypeInfo<Tuple3<Integer, String, Double>>) TypeExtractor.getForObject(multiTupleTest);
    ExpressionKeys fieldKey = new ExpressionKeys(new int[] { 1, 0, 2 }, multiTupleType);
    ExpressionKeys expressKey = new ExpressionKeys(new String[] { "nestedClass.dumm2", "nestedClass.dumm1", "nestedClass.dumm3" }, pType);
    try {
        Assert.assertTrue("Expecting the keys to be compatible", fieldKey.areCompatible(expressKey));
    } catch (IncompatibleKeysException e) {
        e.printStackTrace();
        Assert.fail("Keys must be compatible: " + e.getMessage());
    }
    TypeComparator<TestUserClass> multiPojoComp = pType.createComparator(expressKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiPojoHash = multiPojoComp.hash(pojoTestRecord);
    // pojo order is: dumm2 (str), dumm1 (int), dumm3 (double).
    TypeComparator<Tuple3<Integer, String, Double>> multiTupleComp = multiTupleType.createComparator(fieldKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiTupleHash = multiTupleComp.hash(multiTupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable. Also for those with multiple key fields", multiPojoHash == multiTupleHash);
}

Example 32 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class TupleSerializerTest method runTests.

private <T extends Tuple> void runTests(int length, T... instances) {
    try {
        TupleTypeInfo<T> tupleTypeInfo = (TupleTypeInfo<T>) TypeExtractor.getForObject(instances[0]);
        TypeSerializer<T> serializer = tupleTypeInfo.createSerializer(new ExecutionConfig());
        Class<T> tupleClass = tupleTypeInfo.getTypeClass();
        if (tupleClass == Tuple0.class) {
            length = 1;
        }
        TupleSerializerTestInstance<T> test = new TupleSerializerTestInstance<T>(serializer, tupleClass, length, instances);
        test.testAll();
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 33 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class DistinctTranslationTest method translateDistinctKeySelector.

@Test
public void translateDistinctKeySelector() {
    try {
        final int parallelism = 8;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);
        initialData.distinct(new KeySelector<Tuple3<Double, StringValue, LongValue>, StringValue>() {

            public StringValue getKey(Tuple3<Double, StringValue, LongValue> value) {
                return value.f1;
            }
        }).setParallelism(4).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        MapOperatorBase<?, ?, ?> keyRemover = (MapOperatorBase<?, ?, ?>) sink.getInput();
        PlanUnwrappingReduceOperator<?, ?> reducer = (PlanUnwrappingReduceOperator<?, ?>) keyRemover.getInput();
        MapOperatorBase<?, ?, ?> keyExtractor = (MapOperatorBase<?, ?, ?>) reducer.getInput();
        // check the parallelisms
        assertEquals(1, keyExtractor.getParallelism());
        assertEquals(4, reducer.getParallelism());
        // check types
        TypeInformation<?> keyValueInfo = new TupleTypeInfo<Tuple2<StringValue, Tuple3<Double, StringValue, LongValue>>>(new ValueTypeInfo<StringValue>(StringValue.class), initialData.getType());
        assertEquals(initialData.getType(), keyExtractor.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, keyExtractor.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, keyRemover.getOperatorInfo().getInputType());
        assertEquals(initialData.getType(), keyRemover.getOperatorInfo().getOutputType());
        // check keys
        assertEquals(KeyExtractingMapper.class, keyExtractor.getUserCodeWrapper().getUserCodeClass());
        assertTrue(keyExtractor.getInput() instanceof GenericDataSourceBase<?, ?>);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}

Example 34 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class FieldAccessorFactory method getAccessor.

/**
	 * Creates a {@link FieldAccessor} for the field that is given by a field expression,
	 * which can be used to get and set the specified field on instances of this type.
	 *
	 * @param field The field expression
	 * @param config Configuration object
	 * @param <F> The type of the field to access
	 * @return The created FieldAccessor
	 */
@Internal
public static <T, F> FieldAccessor<T, F> getAccessor(TypeInformation<T> typeInfo, String field, ExecutionConfig config) {
    // In case of arrays
    if (typeInfo instanceof BasicArrayTypeInfo || typeInfo instanceof PrimitiveArrayTypeInfo) {
        try {
            return new FieldAccessor.ArrayFieldAccessor<>(Integer.parseInt(field), typeInfo);
        } catch (NumberFormatException ex) {
            throw new CompositeType.InvalidFieldReferenceException("A field expression on an array must be an integer index (that might be given as a string).");
        }
    // In case of basic types
    } else if (typeInfo instanceof BasicTypeInfo) {
        try {
            int pos = field.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR) ? 0 : Integer.parseInt(field);
            return FieldAccessorFactory.getAccessor(typeInfo, pos, config);
        } catch (NumberFormatException ex) {
            throw new CompositeType.InvalidFieldReferenceException("You tried to select the field \"" + field + "\" on a " + typeInfo.toString() + ". A field expression on a basic type can only be \"*\" or \"0\"" + " (both of which mean selecting the entire basic type).");
        }
    // In case of Pojos
    } else if (typeInfo instanceof PojoTypeInfo) {
        FieldExpression decomp = decomposeFieldExpression(field);
        PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo) typeInfo;
        int fieldIndex = pojoTypeInfo.getFieldIndex(decomp.head);
        if (fieldIndex == -1) {
            throw new CompositeType.InvalidFieldReferenceException("Unable to find field \"" + decomp.head + "\" in type " + typeInfo + ".");
        } else {
            PojoField pojoField = pojoTypeInfo.getPojoFieldAt(fieldIndex);
            TypeInformation<?> fieldType = pojoTypeInfo.getTypeAt(fieldIndex);
            if (decomp.tail == null) {
                @SuppressWarnings("unchecked") FieldAccessor<F, F> innerAccessor = new FieldAccessor.SimpleFieldAccessor<>((TypeInformation<F>) fieldType);
                return new FieldAccessor.PojoFieldAccessor<>(pojoField.getField(), innerAccessor);
            } else {
                @SuppressWarnings("unchecked") FieldAccessor<Object, F> innerAccessor = FieldAccessorFactory.getAccessor((TypeInformation<Object>) fieldType, decomp.tail, config);
                return new FieldAccessor.PojoFieldAccessor<>(pojoField.getField(), innerAccessor);
            }
        }
    // In case of case classes
    } else if (typeInfo.isTupleType() && ((TupleTypeInfoBase) typeInfo).isCaseClass()) {
        TupleTypeInfoBase tupleTypeInfo = (TupleTypeInfoBase) typeInfo;
        FieldExpression decomp = decomposeFieldExpression(field);
        int fieldPos = tupleTypeInfo.getFieldIndex(decomp.head);
        if (fieldPos < 0) {
            throw new CompositeType.InvalidFieldReferenceException("Invalid field selected: " + field);
        }
        if (decomp.tail == null) {
            return new FieldAccessor.SimpleProductFieldAccessor<>(fieldPos, typeInfo, config);
        } else {
            @SuppressWarnings("unchecked") FieldAccessor<Object, F> innerAccessor = getAccessor(tupleTypeInfo.getTypeAt(fieldPos), decomp.tail, config);
            return new FieldAccessor.RecursiveProductFieldAccessor<>(fieldPos, typeInfo, innerAccessor, config);
        }
    // In case of tuples
    } else if (typeInfo.isTupleType()) {
        TupleTypeInfo tupleTypeInfo = (TupleTypeInfo) typeInfo;
        FieldExpression decomp = decomposeFieldExpression(field);
        int fieldPos = tupleTypeInfo.getFieldIndex(decomp.head);
        if (fieldPos == -1) {
            try {
                fieldPos = Integer.parseInt(decomp.head);
            } catch (NumberFormatException ex) {
                throw new CompositeType.InvalidFieldReferenceException("Tried to select field \"" + decomp.head + "\" on " + typeInfo.toString() + " . Only integer values are allowed here.");
            }
        }
        if (decomp.tail == null) {
            @SuppressWarnings("unchecked") FieldAccessor<T, F> result = new FieldAccessor.SimpleTupleFieldAccessor(fieldPos, tupleTypeInfo);
            return result;
        } else {
            @SuppressWarnings("unchecked") FieldAccessor<?, F> innerAccessor = getAccessor(tupleTypeInfo.getTypeAt(fieldPos), decomp.tail, config);
            @SuppressWarnings("unchecked") FieldAccessor<T, F> result = new FieldAccessor.RecursiveTupleFieldAccessor(fieldPos, innerAccessor, tupleTypeInfo);
            return result;
        }
    // Default statement
    } else {
        throw new CompositeType.InvalidFieldReferenceException("Cannot reference field by field expression on " + typeInfo.toString() + "Field expressions are only supported on POJO types, tuples, and case classes. " + "(See the Flink documentation on what is considered a POJO.)");
    }
}

Example 35 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class DataStreamTest method testTupleNestedArrayKeyRejection.

////////////////			Composite Key Tests : Tuples			////////////////
@Test
public void testTupleNestedArrayKeyRejection() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<Integer[], String>> input = env.fromElements(new Tuple2<>(new Integer[] { 1, 2 }, "test-test"));
    TypeInformation<?> expectedTypeInfo = new TupleTypeInfo<Tuple2<Integer[], String>>(BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    // adjust the rule
    expectedException.expect(InvalidProgramException.class);
    expectedException.expectMessage(new StringStartsWith("Type " + expectedTypeInfo + " cannot be used as key."));
    input.keyBy(new KeySelector<Tuple2<Integer[], String>, Tuple2<Integer[], String>>() {

        @Override
        public Tuple2<Integer[], String> getKey(Tuple2<Integer[], String> value) throws Exception {
            return value;
        }
    });
}