use of org.apache.flink.formats.avro.generated.User in project flink by apache.
the class AvroTypesITCase method testAvroObjectAccess.
@Test
public void testAvroObjectAccess() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
DataStream<User> ds = testData(env);
Table t = tEnv.fromDataStream(ds, selectFields(ds));
Table result = t.filter($("type_nested").isNotNull()).select($("type_nested").flatten()).as("city", "num", "state", "street", "zip");
List<Address> results = CollectionUtil.iteratorToList(DataStreamUtils.collect(tEnv.toAppendStream(result, Address.class)));
String expected = USER_1.getTypeNested().toString();
TestBaseUtils.compareResultAsText(results, expected);
}
use of org.apache.flink.formats.avro.generated.User in project flink by apache.
the class AvroTypesITCase method testAvroToRow.
@Test
public void testAvroToRow() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
DataStream<User> ds = testData(env);
Table t = tEnv.fromDataStream(ds, selectFields(ds));
Table result = t.select($("*"));
List<Row> results = CollectionUtil.iteratorToList(DataStreamUtils.collect(tEnv.toAppendStream(result, Row.class)));
// TODO we should get an Avro record here instead of a nested row.
// This should be fixed with FLIP-136
String expected = "+I[black, null, Whatever, [true], [hello], true, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10], " + "2014-03-01, java.nio.HeapByteBuffer[pos=0 lim=2 cap=2], [7, -48], 0.0, GREEN, " + "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 42, {}, null, null, null, 00:00:00.123456, " + "12:12:12, 1970-01-01T00:00:00.123456Z, 2014-03-01T12:12:12.321Z, null]\n" + "+I[blue, null, Charlie, [], [], false, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10], 2014-03-01, " + "java.nio.HeapByteBuffer[pos=0 lim=2 cap=2], [7, -48], 1.337, RED, null, 1337, {}, " + "+I[Berlin, 42, Berlin, Bakerstreet, 12049], null, null, 00:00:00.123456, 12:12:12, 1970-01-01T00:00:00.123456Z, " + "2014-03-01T12:12:12.321Z, null]\n" + "+I[yellow, null, Terminator, [false], [world], false, " + "java.nio.HeapByteBuffer[pos=0 lim=10 cap=10], 2014-03-01, " + "java.nio.HeapByteBuffer[pos=0 lim=2 cap=2], [7, -48], 0.0, GREEN, " + "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 1, {}, null, null, null, 00:00:00.123456, " + "12:12:12, 1970-01-01T00:00:00.123456Z, 2014-03-01T12:12:12.321Z, null]";
TestBaseUtils.compareResultAsText(results, expected);
}
use of org.apache.flink.formats.avro.generated.User in project flink by apache.
the class AvroSerializerTest method getTestData.
@Override
protected User[] getTestData() {
final Random rnd = new Random();
final User[] users = new User[20];
for (int i = 0; i < users.length; i++) {
users[i] = TestDataGenerator.generateRandomUser(rnd);
}
return users;
}
use of org.apache.flink.formats.avro.generated.User in project flink by apache.
the class AvroTypeExtractionTest method testField.
private void testField(final String fieldName) throws Exception {
before();
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
Path in = new Path(inFile.getAbsoluteFile().toURI());
AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
DataSet<User> usersDS = env.createInput(users);
DataSet<Object> res = usersDS.groupBy(fieldName).reduceGroup((GroupReduceFunction<User, Object>) (values, out) -> {
for (User u : values) {
out.collect(u.get(fieldName));
}
}).returns(Object.class);
res.writeAsText(resultPath);
env.execute("Simple Avro read job");
// test if automatic registration of the Types worked
ExecutionConfig ec = env.getConfig();
Assert.assertTrue(ec.getRegisteredKryoTypes().contains(Fixed16.class));
switch(fieldName) {
case "name":
expected = "Alyssa\nCharlie";
break;
case "type_enum":
expected = "GREEN\nRED\n";
break;
case "type_double_test":
expected = "123.45\n1.337\n";
break;
default:
Assert.fail("Unknown field");
break;
}
after();
}
use of org.apache.flink.formats.avro.generated.User in project flink by apache.
the class AvroTypeExtractionTest method testSimpleAvroRead.
@Test
public void testSimpleAvroRead() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
Path in = new Path(inFile.getAbsoluteFile().toURI());
AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
DataSet<User> usersDS = env.createInput(users).map((value) -> value);
usersDS.writeAsText(resultPath);
env.execute("Simple Avro read job");
expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, " + "\"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, " + "\"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], " + "\"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", " + "\"type_map\": {\"KEY 2\": 17554, \"KEY 1\": 8546456}, \"type_fixed\": null, \"type_union\": null, " + "\"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", " + "\"state\": \"London\", \"zip\": \"NW1 6XE\"}, " + "\"type_bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\", " + "\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12, \"type_time_micros\": 00:00:00.123456, " + "\"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, " + "\"type_timestamp_micros\": 1970-01-01T00:00:00.123456Z, \"type_decimal_bytes\": \"\\u0007Ð\", " + "\"type_decimal_fixed\": [7, -48]}\n" + "{\"name\": \"Charlie\", \"favorite_number\": null, " + "\"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, " + "\"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], " + "\"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {}, " + "\"type_fixed\": null, \"type_union\": null, " + "\"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", " + "\"zip\": \"NW1 6XE\"}, " + "\"type_bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\", " + "\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12, \"type_time_micros\": 00:00:00.123456, " + "\"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, " + "\"type_timestamp_micros\": 1970-01-01T00:00:00.123456Z, \"type_decimal_bytes\": \"\\u0007Ð\", " + "\"type_decimal_fixed\": [7, -48]}\n";
}
Aggregations