Examples with KeySelector org.apache.flink.api.java.functions.KeySelector used on opensource projects

Example 41 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class WindowTranslationTest method testMergingAssignerWithNonMergingTriggerFails.

// ------------------------------------------------------------------------
// Merging Windows Support
// ------------------------------------------------------------------------
@Test
public void testMergingAssignerWithNonMergingTriggerFails() throws Exception {
    // verify that we check for trigger compatibility
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    WindowedStream<String, String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao").keyBy(new KeySelector<String, String>() {

        private static final long serialVersionUID = 598309916882894293L;

        @Override
        public String getKey(String value) throws Exception {
            return value;
        }
    }).window(EventTimeSessionWindows.withGap(Time.seconds(5)));
    try {
        windowedStream.trigger(new Trigger<String, TimeWindow>() {

            private static final long serialVersionUID = 6558046711583024443L;

            @Override
            public TriggerResult onElement(String element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
                return null;
            }

            @Override
            public TriggerResult onProcessingTime(long time, TimeWindow window, TriggerContext ctx) throws Exception {
                return null;
            }

            @Override
            public TriggerResult onEventTime(long time, TimeWindow window, TriggerContext ctx) throws Exception {
                return null;
            }

            @Override
            public boolean canMerge() {
                return false;
            }

            @Override
            public void clear(TimeWindow window, TriggerContext ctx) throws Exception {
            }
        });
    } catch (UnsupportedOperationException e) {
        // use a catch to ensure that the exception is thrown by the fold
        return;
    }
    fail("The trigger call should fail.");
}

Example 42 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class BroadcastStateITCase method testKeyedWithBroadcastTranslation.

@Test
public void testKeyedWithBroadcastTranslation() throws Exception {
    final MapStateDescriptor<Long, String> utterDescriptor = new MapStateDescriptor<>("broadcast-state", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final Map<Long, String> expected = new HashMap<>();
    expected.put(0L, "test:0");
    expected.put(1L, "test:1");
    expected.put(2L, "test:2");
    expected.put(3L, "test:3");
    expected.put(4L, "test:4");
    expected.put(5L, "test:5");
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        private static final long serialVersionUID = -8500904795760316195L;

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromCollection(expected.values()).assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        private static final long serialVersionUID = -2148318224248467213L;

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    final BroadcastStream<String> broadcast = srcTwo.broadcast(utterDescriptor);
    // the timestamp should be high enough to trigger the timer after all the elements arrive.
    final DataStream<String> output = srcOne.connect(broadcast).process(new TestKeyedBroadcastProcessFunction(100000L, expected));
    output.addSink(new TestSink(expected.size())).setParallelism(1);
    env.execute();
}

Example 43 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class AbstractStreamOperatorTest method testIdleWatermarkHandling.

@Test
public void testIdleWatermarkHandling() throws Exception {
    final WatermarkTestingOperator testOperator = new WatermarkTestingOperator();
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    KeySelector<Long, Integer> dummyKeySelector = l -> 0;
    try (KeyedTwoInputStreamOperatorTestHarness<Integer, Long, Long, Long> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>(testOperator, dummyKeySelector, dummyKeySelector, BasicTypeInfo.INT_TYPE_INFO)) {
        testHarness.setup();
        testHarness.open();
        testHarness.processElement1(1L, 1L);
        testHarness.processElement1(3L, 3L);
        testHarness.processElement1(4L, 4L);
        testHarness.processWatermark1(new Watermark(1L));
        assertThat(testHarness.getOutput(), empty());
        testHarness.processWatermarkStatus2(WatermarkStatus.IDLE);
        expectedOutput.add(new StreamRecord<>(1L));
        expectedOutput.add(new Watermark(1L));
        TestHarnessUtil.assertOutputEquals("Output was not correct", expectedOutput, testHarness.getOutput());
        testHarness.processWatermark1(new Watermark(3L));
        expectedOutput.add(new StreamRecord<>(3L));
        expectedOutput.add(new Watermark(3L));
        TestHarnessUtil.assertOutputEquals("Output was not correct", expectedOutput, testHarness.getOutput());
        testHarness.processWatermarkStatus2(WatermarkStatus.ACTIVE);
        // the other input is active now, we should not emit the watermark
        testHarness.processWatermark1(new Watermark(4L));
        TestHarnessUtil.assertOutputEquals("Output was not correct", expectedOutput, testHarness.getOutput());
    }
}

Example 44 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CoGroupITCase method testCoGroupWithMultipleKeyFieldsWithInnerClassKeyExtractorWithoutClosureCleaner.

@Test
public void testCoGroupWithMultipleKeyFieldsWithInnerClassKeyExtractorWithoutClosureCleaner() throws Exception {
    /*
         * CoGroup with multiple key fields, test that disabling closure cleaner leads to an exception when using inner
         * classes.
         */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableClosureCleaner();
    DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds1 = CollectionDataSets.get5TupleDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
    boolean correctExceptionTriggered = false;
    try {
        DataSet<Tuple3<Integer, Long, String>> coGrouped = ds1.coGroup(ds2).where(new KeySelector<Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, Long>>() {

            @Override
            public Tuple2<Integer, Long> getKey(Tuple5<Integer, Long, Integer, String, Long> t) throws Exception {
                return new Tuple2<Integer, Long>(t.f0, t.f4);
            }
        }).equalTo(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() {

            @Override
            public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> t) {
                return new Tuple2<Integer, Long>(t.f0, t.f1);
            }
        }).with(new CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>() {

            @Override
            public void coGroup(Iterable<Tuple5<Integer, Long, Integer, String, Long>> first, Iterable<Tuple3<Integer, Long, String>> second, Collector<Tuple3<Integer, Long, String>> out) {
                List<String> strs = new ArrayList<String>();
                for (Tuple5<Integer, Long, Integer, String, Long> t : first) {
                    strs.add(t.f3);
                }
                for (Tuple3<Integer, Long, String> t : second) {
                    for (String s : strs) {
                        out.collect(new Tuple3<Integer, Long, String>(t.f0, t.f1, s));
                    }
                }
            }
        });
    } catch (InvalidProgramException ex) {
        correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException);
    }
    Assert.assertTrue(correctExceptionTriggered);
}

Example 45 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class FirstNITCase method testFaultyCast.

/**
 * Test for FLINK-2135.
 */
@Test
public void testFaultyCast() throws Exception {
    ExecutionEnvironment ee = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> b = ee.fromElements("a", "b");
    GroupReduceOperator<String, String> a = b.groupBy(new KeySelector<String, Long>() {

        @Override
        public Long getKey(String value) throws Exception {
            return 1L;
        }
    }).sortGroup(new KeySelector<String, Double>() {

        @Override
        public Double getKey(String value) throws Exception {
            return 1.0;
        }
    }, Order.DESCENDING).first(1);
    List<String> result = b.collect();
    String expected = "a\nb";
    compareResultAsText(result, expected);
}