Examples with AvroInputFormat org.apache.flink.api.java.io.AvroInputFormat used on opensource projects

Example 11 with AvroInputFormat

use of org.apache.flink.api.java.io.AvroInputFormat in project flink by apache.

the class AvroPojoTest method testField.

private void testField(final String fieldName) throws Exception {
    before();
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Object> res = usersDS.groupBy(fieldName).reduceGroup(new GroupReduceFunction<User, Object>() {

        @Override
        public void reduce(Iterable<User> values, Collector<Object> out) throws Exception {
            for (User u : values) {
                out.collect(u.get(fieldName));
            }
        }
    });
    res.writeAsText(resultPath);
    env.execute("Simple Avro read job");
    // test if automatic registration of the Types worked
    ExecutionConfig ec = env.getConfig();
    Assert.assertTrue(ec.getRegisteredKryoTypes().contains(org.apache.flink.api.io.avro.generated.Fixed16.class));
    if (fieldName.equals("name")) {
        expected = "Alyssa\nCharlie";
    } else if (fieldName.equals("type_enum")) {
        expected = "GREEN\nRED\n";
    } else if (fieldName.equals("type_double_test")) {
        expected = "123.45\n1.337\n";
    } else {
        Assert.fail("Unknown field");
    }
    after();
}

Example 12 with AvroInputFormat

use of org.apache.flink.api.java.io.AvroInputFormat in project flink by apache.

the class AvroPojoTest method testWithAvroGenericSer.

@Test
public void testWithAvroGenericSer() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableForceAvro();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Tuple2<String, Integer>> res = usersDS.groupBy(new KeySelector<User, String>() {

        @Override
        public String getKey(User value) throws Exception {
            return String.valueOf(value.getName());
        }
    }).reduceGroup(new GroupReduceFunction<User, Tuple2<String, Integer>>() {

        @Override
        public void reduce(Iterable<User> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (User u : values) {
                out.collect(new Tuple2<String, Integer>(u.getName().toString(), 1));
            }
        }
    });
    res.writeAsText(resultPath);
    env.execute("Avro Key selection");
    expected = "(Charlie,1)\n(Alyssa,1)\n";
}

Example 13 with AvroInputFormat

use of org.apache.flink.api.java.io.AvroInputFormat in project flink by apache.

the class AvroPojoTest method testWithKryoGenericSer.

@Test
public void testWithKryoGenericSer() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableForceKryo();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Tuple2<String, Integer>> res = usersDS.groupBy(new KeySelector<User, String>() {

        @Override
        public String getKey(User value) throws Exception {
            return String.valueOf(value.getName());
        }
    }).reduceGroup(new GroupReduceFunction<User, Tuple2<String, Integer>>() {

        @Override
        public void reduce(Iterable<User> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (User u : values) {
                out.collect(new Tuple2<String, Integer>(u.getName().toString(), 1));
            }
        }
    });
    res.writeAsText(resultPath);
    env.execute("Avro Key selection");
    expected = "(Charlie,1)\n(Alyssa,1)\n";
}