Examples with TupleTypeInfo org.apache.flink.api.java.typeutils.TupleTypeInfo used on opensource projects

Example 46 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class DataStreamTest method testTupleNestedArrayKeyRejection.

// //////////////			Composite Key Tests : Tuples			////////////////
@Test
public void testTupleNestedArrayKeyRejection() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<Integer[], String>> input = env.fromElements(new Tuple2<>(new Integer[] { 1, 2 }, "test-test"));
    TypeInformation<?> expectedTypeInfo = new TupleTypeInfo<Tuple2<Integer[], String>>(BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    // adjust the rule
    expectedException.expect(InvalidProgramException.class);
    expectedException.expectMessage(new StringStartsWith("Type " + expectedTypeInfo + " cannot be used as key."));
    input.keyBy(new KeySelector<Tuple2<Integer[], String>, Tuple2<Integer[], String>>() {

        @Override
        public Tuple2<Integer[], String> getKey(Tuple2<Integer[], String> value) throws Exception {
            return value;
        }
    });
}

Example 47 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class FieldAccessorFactory method getAccessor.

/**
 * Creates a {@link FieldAccessor} for the field that is given by a field expression, which can
 * be used to get and set the specified field on instances of this type.
 *
 * @param field The field expression
 * @param config Configuration object
 * @param <F> The type of the field to access
 * @return The created FieldAccessor
 */
@Internal
public static <T, F> FieldAccessor<T, F> getAccessor(TypeInformation<T> typeInfo, String field, ExecutionConfig config) {
    // In case of arrays
    if (typeInfo instanceof BasicArrayTypeInfo || typeInfo instanceof PrimitiveArrayTypeInfo) {
        try {
            return new FieldAccessor.ArrayFieldAccessor<>(Integer.parseInt(field), typeInfo);
        } catch (NumberFormatException ex) {
            throw new CompositeType.InvalidFieldReferenceException("A field expression on an array must be an integer index (that might be given as a string).");
        }
    // In case of basic types
    } else if (typeInfo instanceof BasicTypeInfo) {
        try {
            int pos = field.equals(Keys.ExpressionKeys.SELECT_ALL_CHAR) ? 0 : Integer.parseInt(field);
            return FieldAccessorFactory.getAccessor(typeInfo, pos, config);
        } catch (NumberFormatException ex) {
            throw new CompositeType.InvalidFieldReferenceException("You tried to select the field \"" + field + "\" on a " + typeInfo.toString() + ". A field expression on a basic type can only be \"*\" or \"0\"" + " (both of which mean selecting the entire basic type).");
        }
    // In case of Pojos
    } else if (typeInfo instanceof PojoTypeInfo) {
        FieldExpression decomp = decomposeFieldExpression(field);
        PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo) typeInfo;
        int fieldIndex = pojoTypeInfo.getFieldIndex(decomp.head);
        if (fieldIndex == -1) {
            throw new CompositeType.InvalidFieldReferenceException("Unable to find field \"" + decomp.head + "\" in type " + typeInfo + ".");
        } else {
            PojoField pojoField = pojoTypeInfo.getPojoFieldAt(fieldIndex);
            TypeInformation<?> fieldType = pojoTypeInfo.getTypeAt(fieldIndex);
            if (decomp.tail == null) {
                @SuppressWarnings("unchecked") FieldAccessor<F, F> innerAccessor = new FieldAccessor.SimpleFieldAccessor<>((TypeInformation<F>) fieldType);
                return new FieldAccessor.PojoFieldAccessor<>(pojoField.getField(), innerAccessor);
            } else {
                @SuppressWarnings("unchecked") FieldAccessor<Object, F> innerAccessor = FieldAccessorFactory.getAccessor((TypeInformation<Object>) fieldType, decomp.tail, config);
                return new FieldAccessor.PojoFieldAccessor<>(pojoField.getField(), innerAccessor);
            }
        }
    // In case of case classes
    } else if (typeInfo.isTupleType() && ((TupleTypeInfoBase) typeInfo).isCaseClass()) {
        TupleTypeInfoBase tupleTypeInfo = (TupleTypeInfoBase) typeInfo;
        FieldExpression decomp = decomposeFieldExpression(field);
        int fieldPos = tupleTypeInfo.getFieldIndex(decomp.head);
        if (fieldPos < 0) {
            throw new CompositeType.InvalidFieldReferenceException("Invalid field selected: " + field);
        }
        if (decomp.tail == null) {
            if (scalaProductFieldAccessorFactory != null) {
                return scalaProductFieldAccessorFactory.createSimpleProductFieldAccessor(fieldPos, typeInfo, config);
            } else {
                throw new IllegalStateException("Scala products are used but Scala API is not on the classpath.");
            }
        } else {
            @SuppressWarnings("unchecked") FieldAccessor<Object, F> innerAccessor = getAccessor(tupleTypeInfo.getTypeAt(fieldPos), decomp.tail, config);
            if (scalaProductFieldAccessorFactory != null) {
                return scalaProductFieldAccessorFactory.createRecursiveProductFieldAccessor(fieldPos, typeInfo, innerAccessor, config);
            } else {
                throw new IllegalStateException("Scala products are used but Scala API is not on the classpath.");
            }
        }
    // In case of tuples
    } else if (typeInfo.isTupleType() && typeInfo instanceof TupleTypeInfo) {
        TupleTypeInfo tupleTypeInfo = (TupleTypeInfo) typeInfo;
        FieldExpression decomp = decomposeFieldExpression(field);
        int fieldPos = tupleTypeInfo.getFieldIndex(decomp.head);
        if (fieldPos == -1) {
            try {
                fieldPos = Integer.parseInt(decomp.head);
            } catch (NumberFormatException ex) {
                throw new CompositeType.InvalidFieldReferenceException("Tried to select field \"" + decomp.head + "\" on " + typeInfo.toString() + " . Only integer values are allowed here.");
            }
        }
        if (decomp.tail == null) {
            @SuppressWarnings("unchecked") FieldAccessor<T, F> result = new FieldAccessor.SimpleTupleFieldAccessor(fieldPos, tupleTypeInfo);
            return result;
        } else {
            @SuppressWarnings("unchecked") FieldAccessor<?, F> innerAccessor = getAccessor(tupleTypeInfo.getTypeAt(fieldPos), decomp.tail, config);
            @SuppressWarnings("unchecked") FieldAccessor<T, F> result = new FieldAccessor.RecursiveTupleFieldAccessor(fieldPos, innerAccessor, tupleTypeInfo);
            return result;
        }
    // Default statement
    } else {
        throw new CompositeType.InvalidFieldReferenceException("Cannot reference field by field expression on " + typeInfo.toString() + "Field expressions are only supported on POJO types, tuples, and case classes. " + "(See the Flink documentation on what is considered a POJO.)");
    }
}

Example 48 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class SourceStreamTaskTest method testCheckpointing.

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing and element
 * emission. This also verifies that there are no concurrent invocations of the checkpoint
 * method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the source
 * kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
    final int numElements = 100;
    final int numCheckpoints = 100;
    final int numCheckpointers = 1;
    // in ms
    final int checkpointInterval = 5;
    final int sourceCheckpointDelay = // how many random values we sum up in storeCheckpoint
    1000;
    // in ms
    final int sourceReadDelay = 1;
    ExecutorService executor = Executors.newFixedThreadPool(10);
    try {
        final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
        final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, typeInfo);
        testHarness.setupOutputForSingletonOperatorChain();
        StreamConfig streamConfig = testHarness.getStreamConfig();
        StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
        streamConfig.setStreamOperator(sourceOperator);
        streamConfig.setOperatorID(new OperatorID());
        // prepare the
        Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];
        // invoke this first, so the tasks are actually running when the checkpoints are
        // scheduled
        testHarness.invoke();
        testHarness.waitForTaskRunning();
        final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();
        for (int i = 0; i < numCheckpointers; i++) {
            checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
        }
        testHarness.waitForTaskCompletion();
        // will be rethrown here
        for (int i = 0; i < numCheckpointers; i++) {
            if (!checkpointerResults[i].isDone()) {
                checkpointerResults[i].cancel(true);
            }
            if (!checkpointerResults[i].isCancelled()) {
                checkpointerResults[i].get();
            }
        }
        List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
        Assert.assertEquals(numElements, resultElements.size());
    } finally {
        executor.shutdown();
    }
}

Example 49 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class ReduceCombineDriverTest method testImmutableEmpty.

@Test
public void testImmutableEmpty() {
    try {
        TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>(1024 * 1024);
        context.getTaskConfig().setRelativeMemoryDriver(0.5);
        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        Collections.shuffle(data);
        TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
        context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
        TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setComparator1(comparator);
        context.setCollector(result);
        ReduceCombineDriver<Tuple2<String, Integer>> driver = new ReduceCombineDriver<Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Assert.assertEquals(0, result.getList().size());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 50 with TupleTypeInfo

use of org.apache.flink.api.java.typeutils.TupleTypeInfo in project flink by apache.

the class GroupReduceDriverTest method testAllReduceDriverMutable.

@Test
public void testAllReduceDriverMutable() {
    try {
        TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
        List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
        TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
        TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setComparator1(comparator);
        context.setCollector(result);
        context.setUdf(new ConcatSumMutableReducer());
        GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Object[] res = result.getList().toArray();
        Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
        DriverTestData.compareTupleArrays(expected, res);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}