Examples with KeySelector org.apache.flink.api.java.functions.KeySelector used on opensource projects

Example 11 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class JoinOperatorTest method testJoinKeySelectors1.

@Test
public void testJoinKeySelectors1() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
    DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
    // should work
    try {
        ds1.join(ds2).where(new KeySelector<CustomType, Long>() {

            @Override
            public Long getKey(CustomType value) {
                return value.myLong;
            }
        }).equalTo(new KeySelector<CustomType, Long>() {

            @Override
            public Long getKey(CustomType value) {
                return value.myLong;
            }
        });
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 12 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class GroupingTest method testGroupSortByKeySelector2.

@SuppressWarnings("serial")
@Test(expected = InvalidProgramException.class)
public void testGroupSortByKeySelector2() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);
    // should not work
    tupleDs.groupBy(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Long>() {

        @Override
        public Long getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
            return value.f1;
        }
    }).sortGroup(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, CustomType>() {

        @Override
        public CustomType getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
            return value.f2;
        }
    }, Order.ASCENDING);
}

Example 13 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class JoinOperatorTest method testJoinProjection6.

@Test
public void testJoinProjection6() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
    DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
    // should work
    try {
        ds1.join(ds2).where(new KeySelector<CustomType, Long>() {

            @Override
            public Long getKey(CustomType value) {
                return value.myLong;
            }
        }).equalTo(new KeySelector<CustomType, Long>() {

            @Override
            public Long getKey(CustomType value) {
                return value.myLong;
            }
        }).projectFirst().projectSecond();
    } catch (Exception e) {
        System.out.println("FAILED: " + e);
        e.printStackTrace();
        Assert.fail();
    }
}

Example 14 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class AvroTypeExtractionTest method testWithKryoGenericSer.

@Test
public void testWithKryoGenericSer() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableForceKryo();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Tuple2<String, Integer>> res = usersDS.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName())).reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
        for (User u : values) {
            out.collect(new Tuple2<>(u.getName().toString(), 1));
        }
    }).returns(Types.TUPLE(Types.STRING, Types.INT));
    res.writeAsText(resultPath);
    env.execute("Avro Key selection");
    expected = "(Charlie,1)\n(Alyssa,1)\n";
}

Example 15 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class QsStateProducer method main.

public static void main(final String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool tool = ParameterTool.fromArgs(args);
    String tmpPath = tool.getRequired("tmp-dir");
    String stateBackendType = tool.getRequired("state-backend");
    StateBackend stateBackend;
    switch(stateBackendType) {
        case "rocksdb":
            stateBackend = new RocksDBStateBackend(tmpPath);
            break;
        case "fs":
            stateBackend = new FsStateBackend(tmpPath);
            break;
        case "memory":
            stateBackend = new MemoryStateBackend();
            break;
        default:
            throw new RuntimeException("Unsupported state backend " + stateBackendType);
    }
    env.setStateBackend(stateBackend);
    env.enableCheckpointing(1000L);
    env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
    env.getCheckpointConfig().setMinPauseBetweenCheckpoints(0);
    env.addSource(new EmailSource()).keyBy(new KeySelector<Email, String>() {

        private static final long serialVersionUID = -1480525724620425363L;

        @Override
        public String getKey(Email value) throws Exception {
            return QsConstants.KEY;
        }
    }).flatMap(new TestFlatMap());
    env.execute();
}