Examples with KeySelector org.apache.flink.api.java.functions.KeySelector used on opensource projects

Example 56 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class AvroPojoTest method testWithKryoGenericSer.

@Test
public void testWithKryoGenericSer() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableForceKryo();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Tuple2<String, Integer>> res = usersDS.groupBy(new KeySelector<User, String>() {

        @Override
        public String getKey(User value) throws Exception {
            return String.valueOf(value.getName());
        }
    }).reduceGroup(new GroupReduceFunction<User, Tuple2<String, Integer>>() {

        @Override
        public void reduce(Iterable<User> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (User u : values) {
                out.collect(new Tuple2<String, Integer>(u.getName().toString(), 1));
            }
        }
    });
    res.writeAsText(resultPath);
    env.execute("Avro Key selection");
    expected = "(Charlie,1)\n(Alyssa,1)\n";
}

Example 57 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class StreamGroupedFoldTest method testGroupedFold.

@Test
public void testGroupedFold() throws Exception {
    KeySelector<Integer, String> keySelector = new KeySelector<Integer, String>() {

        @Override
        public String getKey(Integer value) {
            return value.toString();
        }
    };
    StreamGroupedFold<Integer, String, String> operator = new StreamGroupedFold<>(new MyFolder(), "100");
    operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.STRING_TYPE_INFO);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
    testHarness.processElement(new StreamRecord<>(1, initialTime + 2));
    testHarness.processWatermark(new Watermark(initialTime + 2));
    testHarness.processElement(new StreamRecord<>(2, initialTime + 3));
    testHarness.processElement(new StreamRecord<>(2, initialTime + 4));
    testHarness.processElement(new StreamRecord<>(3, initialTime + 5));
    expectedOutput.add(new StreamRecord<>("1001", initialTime + 1));
    expectedOutput.add(new StreamRecord<>("10011", initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<>("1002", initialTime + 3));
    expectedOutput.add(new StreamRecord<>("10022", initialTime + 4));
    expectedOutput.add(new StreamRecord<>("1003", initialTime + 5));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 58 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class StreamGroupedFoldTest method testOpenClose.

@Test
public void testOpenClose() throws Exception {
    KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {

        @Override
        public Integer getKey(Integer value) {
            return value;
        }
    };
    StreamGroupedFold<Integer, String, Integer> operator = new StreamGroupedFold<>(new TestOpenCloseFoldFunction(), "init");
    operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig());
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO);
    long initialTime = 0L;
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(1, initialTime));
    testHarness.processElement(new StreamRecord<>(2, initialTime));
    testHarness.close();
    assertTrue("RichFunction methods where not called.", TestOpenCloseFoldFunction.closeCalled);
    assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0);
}

Example 59 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class JoinITCase method testDefaultJoinOnTwoCustomTypeInputsWithInnerClassKeyExtractorsDisabledClosureCleaner.

@Test
public void testDefaultJoinOnTwoCustomTypeInputsWithInnerClassKeyExtractorsDisabledClosureCleaner() throws Exception {
    /*
		 * (Default) Join on two custom type inputs with key extractors, check if disableing closure cleaning works
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().disableClosureCleaner();
    DataSet<CustomType> ds1 = CollectionDataSets.getCustomTypeDataSet(env);
    DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
    boolean correctExceptionTriggered = false;
    try {
        DataSet<Tuple2<CustomType, CustomType>> joinDs = ds1.join(ds2).where(new KeySelector<CustomType, Integer>() {

            @Override
            public Integer getKey(CustomType value) {
                return value.myInt;
            }
        }).equalTo(new KeySelector<CustomType, Integer>() {

            @Override
            public Integer getKey(CustomType value) throws Exception {
                return value.myInt;
            }
        });
    } catch (InvalidProgramException ex) {
        correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException);
    }
    Assert.assertTrue(correctExceptionTriggered);
}

Example 60 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class SortPartitionITCase method testSortPartitionWithKeySelector2.

@Test
public void testSortPartitionWithKeySelector2() throws Exception {
    /*
		 * Test sort partition on an extracted key
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    List<Tuple1<Boolean>> result = ds.map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(// parallelize input
    4).sortPartition(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() {

        @Override
        public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> value) throws Exception {
            return new Tuple2<>(value.f0, value.f1);
        }
    }, Order.DESCENDING).mapPartition(new OrderCheckMapper<>(new Tuple3Checker())).distinct().collect();
    String expected = "(true)\n";
    compareResultAsText(result, expected);
}