Examples with TupleTypeInfo org.apache.flink.api.java.typeutils.TupleTypeInfo used on opensource projects

Example 26 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class SemanticPropUtil method createProjectionPropertiesDual.

public static DualInputSemanticProperties createProjectionPropertiesDual(int[] fields, boolean[] isFromFirst, TypeInformation<?> inType1, TypeInformation<?> inType2) {
    DualInputSemanticProperties dsp = new DualInputSemanticProperties();
    int[] sourceOffsets1;
    if (inType1 instanceof TupleTypeInfo<?>) {
        sourceOffsets1 = new int[inType1.getArity()];
        sourceOffsets1[0] = 0;
        for (int i = 1; i < inType1.getArity(); i++) {
            sourceOffsets1[i] = ((TupleTypeInfo<?>) inType1).getTypeAt(i - 1).getTotalFields() + sourceOffsets1[i - 1];
        }
    } else {
        sourceOffsets1 = new int[] { 0 };
    }
    int[] sourceOffsets2;
    if (inType2 instanceof TupleTypeInfo<?>) {
        sourceOffsets2 = new int[inType2.getArity()];
        sourceOffsets2[0] = 0;
        for (int i = 1; i < inType2.getArity(); i++) {
            sourceOffsets2[i] = ((TupleTypeInfo<?>) inType2).getTypeAt(i - 1).getTotalFields() + sourceOffsets2[i - 1];
        }
    } else {
        sourceOffsets2 = new int[] { 0 };
    }
    int targetOffset = 0;
    for (int i = 0; i < fields.length; i++) {
        int sourceOffset;
        int numFieldsToCopy;
        int input;
        if (isFromFirst[i]) {
            input = 0;
            if (fields[i] == -1) {
                sourceOffset = 0;
                numFieldsToCopy = inType1.getTotalFields();
            } else {
                sourceOffset = sourceOffsets1[fields[i]];
                numFieldsToCopy = ((TupleTypeInfo<?>) inType1).getTypeAt(fields[i]).getTotalFields();
            }
        } else {
            input = 1;
            if (fields[i] == -1) {
                sourceOffset = 0;
                numFieldsToCopy = inType2.getTotalFields();
            } else {
                sourceOffset = sourceOffsets2[fields[i]];
                numFieldsToCopy = ((TupleTypeInfo<?>) inType2).getTypeAt(fields[i]).getTotalFields();
            }
        }
        for (int j = 0; j < numFieldsToCopy; j++) {
            dsp.addForwardedField(input, sourceOffset + j, targetOffset + j);
        }
        targetOffset += numFieldsToCopy;
    }
    return dsp;
}

Example 27 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class PojoSerializerTest method testTuplePojoTestEquality.

/**
	 * This tests if the hashes returned by the pojo and tuple comparators are the same
	 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTuplePojoTestEquality() {
    // test with a simple, string-key first.
    PojoTypeInfo<TestUserClass> pType = (PojoTypeInfo<TestUserClass>) type;
    List<FlatFieldDescriptor> result = new ArrayList<FlatFieldDescriptor>();
    pType.getFlatFields("nestedClass.dumm2", 0, result);
    // see below
    int[] fields = new int[1];
    fields[0] = result.get(0).getPosition();
    TypeComparator<TestUserClass> pojoComp = pType.createComparator(fields, new boolean[] { true }, 0, new ExecutionConfig());
    TestUserClass pojoTestRecord = new TestUserClass(0, "abc", 3d, new int[] { 1, 2, 3 }, new Date(), new NestedTestUserClass(1, "haha", 4d, new int[] { 5, 4, 3 }));
    int pHash = pojoComp.hash(pojoTestRecord);
    Tuple1<String> tupleTest = new Tuple1<String>("haha");
    TupleTypeInfo<Tuple1<String>> tType = (TupleTypeInfo<Tuple1<String>>) TypeExtractor.getForObject(tupleTest);
    TypeComparator<Tuple1<String>> tupleComp = tType.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
    int tHash = tupleComp.hash(tupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable", pHash == tHash);
    // its important here to use the same values.
    Tuple3<Integer, String, Double> multiTupleTest = new Tuple3<Integer, String, Double>(1, "haha", 4d);
    TupleTypeInfo<Tuple3<Integer, String, Double>> multiTupleType = (TupleTypeInfo<Tuple3<Integer, String, Double>>) TypeExtractor.getForObject(multiTupleTest);
    ExpressionKeys fieldKey = new ExpressionKeys(new int[] { 1, 0, 2 }, multiTupleType);
    ExpressionKeys expressKey = new ExpressionKeys(new String[] { "nestedClass.dumm2", "nestedClass.dumm1", "nestedClass.dumm3" }, pType);
    try {
        Assert.assertTrue("Expecting the keys to be compatible", fieldKey.areCompatible(expressKey));
    } catch (IncompatibleKeysException e) {
        e.printStackTrace();
        Assert.fail("Keys must be compatible: " + e.getMessage());
    }
    TypeComparator<TestUserClass> multiPojoComp = pType.createComparator(expressKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiPojoHash = multiPojoComp.hash(pojoTestRecord);
    // pojo order is: dumm2 (str), dumm1 (int), dumm3 (double).
    TypeComparator<Tuple3<Integer, String, Double>> multiTupleComp = multiTupleType.createComparator(fieldKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiTupleHash = multiTupleComp.hash(multiTupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable. Also for those with multiple key fields", multiPojoHash == multiTupleHash);
}

Example 28 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class TupleSerializerTest method runTests.

private <T extends Tuple> void runTests(int length, T... instances) {
    try {
        TupleTypeInfo<T> tupleTypeInfo = (TupleTypeInfo<T>) TypeExtractor.getForObject(instances[0]);
        TypeSerializer<T> serializer = tupleTypeInfo.createSerializer(new ExecutionConfig());
        Class<T> tupleClass = tupleTypeInfo.getTypeClass();
        if (tupleClass == Tuple0.class) {
            length = 1;
        }
        TupleSerializerTestInstance<T> test = new TupleSerializerTestInstance<T>(serializer, tupleClass, length, instances);
        test.testAll();
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 29 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class DistinctTranslationTest method translateDistinctKeySelector.

@Test
public void translateDistinctKeySelector() {
    try {
        final int parallelism = 8;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);
        initialData.distinct(new KeySelector<Tuple3<Double, StringValue, LongValue>, StringValue>() {

            public StringValue getKey(Tuple3<Double, StringValue, LongValue> value) {
                return value.f1;
            }
        }).setParallelism(4).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        MapOperatorBase<?, ?, ?> keyRemover = (MapOperatorBase<?, ?, ?>) sink.getInput();
        PlanUnwrappingReduceOperator<?, ?> reducer = (PlanUnwrappingReduceOperator<?, ?>) keyRemover.getInput();
        MapOperatorBase<?, ?, ?> keyExtractor = (MapOperatorBase<?, ?, ?>) reducer.getInput();
        // check the parallelisms
        assertEquals(1, keyExtractor.getParallelism());
        assertEquals(4, reducer.getParallelism());
        // check types
        TypeInformation<?> keyValueInfo = new TupleTypeInfo<Tuple2<StringValue, Tuple3<Double, StringValue, LongValue>>>(new ValueTypeInfo<StringValue>(StringValue.class), initialData.getType());
        assertEquals(initialData.getType(), keyExtractor.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, keyExtractor.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, keyRemover.getOperatorInfo().getInputType());
        assertEquals(initialData.getType(), keyRemover.getOperatorInfo().getOutputType());
        // check keys
        assertEquals(KeyExtractingMapper.class, keyExtractor.getUserCodeWrapper().getUserCodeClass());
        assertTrue(keyExtractor.getInput() instanceof GenericDataSourceBase<?, ?>);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}

Example 30 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class Graph method mapEdges.

/**
	 * Apply a function to the attribute of each edge in the graph.
	 * 
	 * @param mapper the map function to apply.
	 * @return a new graph
	 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public <NV> Graph<K, VV, NV> mapEdges(final MapFunction<Edge<K, EV>, NV> mapper) {
    TypeInformation<K> keyType = ((TupleTypeInfo<?>) edges.getType()).getTypeAt(0);
    TypeInformation<NV> valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, edges.getType(), null);
    TypeInformation<Edge<K, NV>> returnType = (TypeInformation<Edge<K, NV>>) new TupleTypeInfo(Edge.class, keyType, keyType, valueType);
    return mapEdges(mapper, returnType);
}