Examples with Tuple2 org.apache.flink.api.java.tuple.Tuple2 used on opensource projects

Example 46 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class JoinITCase method testNonPojoToVerifyFullTupleKeys.

@Test
public void testNonPojoToVerifyFullTupleKeys() throws Exception {
    /*
		 * Non-POJO test to verify that full-tuple keys are working.
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env);
    DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env);
    DataSet<Tuple2<Tuple2<Tuple2<Integer, Integer>, String>, Tuple2<Tuple2<Integer, Integer>, String>>> joinDs = // key is now Tuple2<Integer, Integer>
    ds1.join(ds2).where(0).equalTo("f0.f0", "f0.f1");
    env.setParallelism(1);
    List<Tuple2<Tuple2<Tuple2<Integer, Integer>, String>, Tuple2<Tuple2<Integer, Integer>, String>>> result = joinDs.collect();
    String expected = "((1,1),one),((1,1),one)\n" + "((2,2),two),((2,2),two)\n" + "((3,3),three),((3,3),three)\n";
    compareResultAsTuples(result, expected);
}

Example 47 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class FirstNITCase method testFirstNOnGroupedAndSortedDS.

@Test
public void testFirstNOnGroupedAndSortedDS() throws Exception {
    /*
		 * First-n on grouped and sorted data set
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple2<Long, Integer>> first = ds.groupBy(1).sortGroup(0, Order.DESCENDING).first(3).project(1, 0);
    List<Tuple2<Long, Integer>> result = first.collect();
    String expected = "(1,1)\n" + "(2,3)\n(2,2)\n" + "(3,6)\n(3,5)\n(3,4)\n" + "(4,10)\n(4,9)\n(4,8)\n" + "(5,15)\n(5,14)\n(5,13)\n" + "(6,21)\n(6,20)\n(6,19)\n";
    compareResultAsText(result, expected);
}

Example 48 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class SerializedCheckpointData method toDeque.

// ------------------------------------------------------------------------
//  De-Serialize from Checkpoint
// ------------------------------------------------------------------------
/**
	 * De-serializes an array of SerializedCheckpointData back into an ArrayDeque of element checkpoints.
	 * 
	 * @param data The data to be deserialized.
	 * @param serializer The serializer used to deserialize the data.
	 * @param <T> The type of the elements.
	 * @return An ArrayDeque of element checkpoints.
	 * 
	 * @throws IOException Thrown, if the serialization fails.
	 */
public static <T> ArrayDeque<Tuple2<Long, List<T>>> toDeque(SerializedCheckpointData[] data, TypeSerializer<T> serializer) throws IOException {
    ArrayDeque<Tuple2<Long, List<T>>> deque = new ArrayDeque<>(data.length);
    DataInputDeserializer deser = null;
    for (SerializedCheckpointData checkpoint : data) {
        byte[] serializedData = checkpoint.getSerializedData();
        if (deser == null) {
            deser = new DataInputDeserializer(serializedData, 0, serializedData.length);
        } else {
            deser.setBuffer(serializedData, 0, serializedData.length);
        }
        final List<T> ids = new ArrayList<>(checkpoint.getNumIds());
        final int numIds = checkpoint.getNumIds();
        for (int i = 0; i < numIds; i++) {
            ids.add(serializer.deserialize(deser));
        }
        deque.addLast(new Tuple2<Long, List<T>>(checkpoint.checkpointId, ids));
    }
    return deque;
}

Example 49 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class WindowOperatorTest method testSlidingEventTimeWindows.

private void testSlidingEventTimeWindows(OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness) throws Exception {
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    // add elements out-of-order
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 999));
    expectedOutput.add(new Watermark(999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.processWatermark(new Watermark(1999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 1999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 1999));
    expectedOutput.add(new Watermark(1999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.processWatermark(new Watermark(2999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
    expectedOutput.add(new Watermark(2999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    // do a snapshot, close and restore again
    OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
    testHarness.close();
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processWatermark(new Watermark(3999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
    expectedOutput.add(new Watermark(3999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.processWatermark(new Watermark(4999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 4999));
    expectedOutput.add(new Watermark(4999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    testHarness.processWatermark(new Watermark(5999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
    expectedOutput.add(new Watermark(5999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
    // those don't have any effect...
    testHarness.processWatermark(new Watermark(6999));
    testHarness.processWatermark(new Watermark(7999));
    expectedOutput.add(new Watermark(6999));
    expectedOutput.add(new Watermark(7999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
}

Example 50 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class WindowOperatorTest method testCleanupTimerWithEmptyFoldingStateForSessionWindows.

// TODO this test seems invalid, as it uses the unsupported combination of merging windows and folding window state
@Test
public void testCleanupTimerWithEmptyFoldingStateForSessionWindows() throws Exception {
    final int GAP_SIZE = 3;
    final long LATENESS = 10;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    FoldingStateDescriptor<Tuple2<String, Integer>, Tuple2<String, Integer>> windowStateDesc = new FoldingStateDescriptor<>("window-contents", new Tuple2<>((String) null, 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator, Tuple2<String, Integer> value) throws Exception {
            return new Tuple2<>(value.f0, accumulator.f1 + value.f1);
        }
    }, inputType);
    windowStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalSingleValueWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), LATENESS, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.open();
    ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
    testHarness.processWatermark(new Watermark(4998));
    expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
    expected.add(new Watermark(4998));
    testHarness.processWatermark(new Watermark(14600));
    expected.add(new Watermark(14600));
    ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
    testHarness.close();
}