Examples with User org.apache.flink.formats.avro.generated.User used on opensource projects

Example 21 with User

use of org.apache.flink.formats.avro.generated.User in project flink by apache.

the class AvroRecordInputFormatTest method testDeserializationReuseAvroRecordFalse.

/**
 * Test if the AvroInputFormat is able to properly read data from an Avro file.
 */
@Test
public void testDeserializationReuseAvroRecordFalse() throws IOException {
    Configuration parameters = new Configuration();
    AvroInputFormat<User> format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);
    format.setReuseAvroValue(false);
    format.configure(parameters);
    FileInputSplit[] splits = format.createInputSplits(1);
    assertEquals(splits.length, 1);
    format.open(splits[0]);
    User u = format.nextRecord(null);
    assertNotNull(u);
    String name = u.getName().toString();
    assertNotNull("empty record", name);
    assertEquals("name not equal", TEST_NAME, name);
    // check arrays
    List<CharSequence> sl = u.getTypeArrayString();
    assertEquals("element 0 not equal", TEST_ARRAY_STRING_1, sl.get(0).toString());
    assertEquals("element 1 not equal", TEST_ARRAY_STRING_2, sl.get(1).toString());
    List<Boolean> bl = u.getTypeArrayBoolean();
    assertEquals("element 0 not equal", TEST_ARRAY_BOOLEAN_1, bl.get(0));
    assertEquals("element 1 not equal", TEST_ARRAY_BOOLEAN_2, bl.get(1));
    // check enums
    Colors enumValue = u.getTypeEnum();
    assertEquals("enum not equal", TEST_ENUM_COLOR, enumValue);
    // check maps
    Map<CharSequence, Long> lm = u.getTypeMap();
    assertEquals("map value of key 1 not equal", TEST_MAP_VALUE1, lm.get(new Utf8(TEST_MAP_KEY1)).longValue());
    assertEquals("map value of key 2 not equal", TEST_MAP_VALUE2, lm.get(new Utf8(TEST_MAP_KEY2)).longValue());
    assertFalse("expecting second element", format.reachedEnd());
    assertNotNull("expecting second element", format.nextRecord(u));
    assertNull(format.nextRecord(u));
    assertTrue(format.reachedEnd());
    format.close();
}

Example 22 with User

use of org.apache.flink.formats.avro.generated.User in project flink by apache.

the class AvroTypeInfoTest method testAvroByDefault.

@Test
public void testAvroByDefault() {
    final TypeSerializer<User> serializer = new AvroTypeInfo<>(User.class).createSerializer(new ExecutionConfig());
    assertTrue(serializer instanceof AvroSerializer);
}

Example 23 with User

use of org.apache.flink.formats.avro.generated.User in project flink by apache.

the class AvroTestUtils method getSpecificTestData.

/**
 * Tests all Avro data types as well as nested types for a specific record.
 */
public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getSpecificTestData() {
    final Address addr = Address.newBuilder().setNum(42).setStreet("Main Street 42").setCity("Test City").setState("Test State").setZip("12345").build();
    final Row rowAddr = new Row(5);
    rowAddr.setField(0, 42);
    rowAddr.setField(1, "Main Street 42");
    rowAddr.setField(2, "Test City");
    rowAddr.setField(3, "Test State");
    rowAddr.setField(4, "12345");
    final User user = User.newBuilder().setName("Charlie").setFavoriteNumber(null).setFavoriteColor("blue").setTypeLongTest(1337L).setTypeDoubleTest(1.337d).setTypeNullTest(null).setTypeBoolTest(false).setTypeArrayString(Arrays.asList("hello", "world")).setTypeArrayBoolean(Arrays.asList(true, true, false)).setTypeNullableArray(null).setTypeEnum(Colors.RED).setTypeMap(Collections.singletonMap("test", 12L)).setTypeFixed(new Fixed16(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 })).setTypeUnion(12.0).setTypeNested(addr).setTypeBytes(ByteBuffer.allocate(10)).setTypeDate(LocalDate.parse("2014-03-01")).setTypeTimeMillis(LocalTime.parse("12:12:12")).setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)).setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z")).setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)).setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).build();
    final Row rowUser = new Row(23);
    rowUser.setField(0, "Charlie");
    rowUser.setField(1, null);
    rowUser.setField(2, "blue");
    rowUser.setField(3, 1337L);
    rowUser.setField(4, 1.337d);
    rowUser.setField(5, null);
    rowUser.setField(6, false);
    rowUser.setField(7, new String[] { "hello", "world" });
    rowUser.setField(8, new Boolean[] { true, true, false });
    rowUser.setField(9, null);
    rowUser.setField(10, "RED");
    rowUser.setField(11, Collections.singletonMap("test", 12L));
    rowUser.setField(12, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 });
    rowUser.setField(13, 12.0);
    rowUser.setField(14, rowAddr);
    rowUser.setField(15, new byte[10]);
    rowUser.setField(16, Date.valueOf("2014-03-01"));
    rowUser.setField(17, Time.valueOf("12:12:12"));
    rowUser.setField(18, Time.valueOf(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(19, Timestamp.valueOf("2014-03-01 12:12:12.321"));
    rowUser.setField(20, Timestamp.from(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(21, BigDecimal.valueOf(2000, 2));
    rowUser.setField(22, BigDecimal.valueOf(2000, 2));
    final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
    t.f0 = User.class;
    t.f1 = user;
    t.f2 = rowUser;
    return t;
}

Example 24 with User

use of org.apache.flink.formats.avro.generated.User in project flink by apache.

the class AvroSplittableInputFormatTest method createFiles.

@Before
public void createFiles() throws IOException {
    testFile = File.createTempFile("AvroSplittableInputFormatTest", null);
    ArrayList<CharSequence> stringArray = new ArrayList<>();
    stringArray.add(TEST_ARRAY_STRING_1);
    stringArray.add(TEST_ARRAY_STRING_2);
    ArrayList<Boolean> booleanArray = new ArrayList<>();
    booleanArray.add(TEST_ARRAY_BOOLEAN_1);
    booleanArray.add(TEST_ARRAY_BOOLEAN_2);
    HashMap<CharSequence, Long> longMap = new HashMap<>();
    longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
    longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
    Address addr = new Address();
    addr.setNum(TEST_NUM);
    addr.setStreet(TEST_STREET);
    addr.setCity(TEST_CITY);
    addr.setState(TEST_STATE);
    addr.setZip(TEST_ZIP);
    User user1 = new User();
    user1.setName(TEST_NAME);
    user1.setFavoriteNumber(256);
    user1.setTypeDoubleTest(123.45d);
    user1.setTypeBoolTest(true);
    user1.setTypeArrayString(stringArray);
    user1.setTypeArrayBoolean(booleanArray);
    user1.setTypeEnum(TEST_ENUM_COLOR);
    user1.setTypeMap(longMap);
    user1.setTypeNested(addr);
    user1.setTypeBytes(ByteBuffer.allocate(10));
    user1.setTypeDate(LocalDate.parse("2014-03-01"));
    user1.setTypeTimeMillis(LocalTime.parse("12:12:12"));
    user1.setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS));
    user1.setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z"));
    user1.setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS));
    // 20.00
    user1.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    // 20.00
    user1.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    // Construct via builder
    User user2 = User.newBuilder().setName(TEST_NAME).setFavoriteColor("blue").setFavoriteNumber(null).setTypeBoolTest(false).setTypeDoubleTest(1.337d).setTypeNullTest(null).setTypeLongTest(1337L).setTypeArrayString(new ArrayList<>()).setTypeArrayBoolean(new ArrayList<>()).setTypeNullableArray(null).setTypeEnum(Colors.RED).setTypeMap(new HashMap<>()).setTypeFixed(new Fixed16()).setTypeUnion(123L).setTypeNested(Address.newBuilder().setNum(TEST_NUM).setStreet(TEST_STREET).setCity(TEST_CITY).setState(TEST_STATE).setZip(TEST_ZIP).build()).setTypeBytes(ByteBuffer.allocate(10)).setTypeDate(LocalDate.parse("2014-03-01")).setTypeTimeMillis(LocalTime.parse("12:12:12")).setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)).setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z")).setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)).setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).build();
    DatumWriter<User> userDatumWriter = new SpecificDatumWriter<>(User.class);
    DataFileWriter<User> dataFileWriter = new DataFileWriter<>(userDatumWriter);
    dataFileWriter.create(user1.getSchema(), testFile);
    dataFileWriter.append(user1);
    dataFileWriter.append(user2);
    Random rnd = new Random(1337);
    for (int i = 0; i < NUM_RECORDS - 2; i++) {
        User user = new User();
        user.setName(TEST_NAME + rnd.nextInt());
        user.setFavoriteNumber(rnd.nextInt());
        user.setTypeDoubleTest(rnd.nextDouble());
        user.setTypeBoolTest(true);
        user.setTypeArrayString(stringArray);
        user.setTypeArrayBoolean(booleanArray);
        user.setTypeEnum(TEST_ENUM_COLOR);
        user.setTypeMap(longMap);
        Address address = new Address();
        address.setNum(TEST_NUM);
        address.setStreet(TEST_STREET);
        address.setCity(TEST_CITY);
        address.setState(TEST_STATE);
        address.setZip(TEST_ZIP);
        user.setTypeNested(address);
        user.setTypeBytes(ByteBuffer.allocate(10));
        user.setTypeDate(LocalDate.parse("2014-03-01"));
        user.setTypeTimeMillis(LocalTime.parse("12:12:12"));
        user.setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS));
        user.setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z"));
        user.setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS));
        // 20.00
        user.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
        // 20.00
        user.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
}

Example 25 with User

use of org.apache.flink.formats.avro.generated.User in project flink by apache.

the class AvroSplittableInputFormatTest method testSplittedIF.

@Test
public void testSplittedIF() throws IOException {
    Configuration parameters = new Configuration();
    AvroInputFormat<User> format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);
    format.configure(parameters);
    FileInputSplit[] splits = format.createInputSplits(4);
    assertEquals(splits.length, 4);
    int elements = 0;
    int[] elementsPerSplit = new int[4];
    for (int i = 0; i < splits.length; i++) {
        format.open(splits[i]);
        while (!format.reachedEnd()) {
            User u = format.nextRecord(null);
            Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
            elements++;
            elementsPerSplit[i]++;
        }
        format.close();
    }
    Assert.assertEquals(1604, elementsPerSplit[0]);
    Assert.assertEquals(1203, elementsPerSplit[1]);
    Assert.assertEquals(1203, elementsPerSplit[2]);
    Assert.assertEquals(990, elementsPerSplit[3]);
    Assert.assertEquals(NUM_RECORDS, elements);
    format.close();
}