Examples with Fixed16 org.apache.flink.formats.avro.generated.Fixed16 used on opensource projects

Example 1 with Fixed16

use of org.apache.flink.formats.avro.generated.Fixed16 in project flink by apache.

the class AvroTestUtils method getGenericTestData.

/**
 * Tests almost all Avro data types as well as nested types for a generic record.
 */
public static Tuple3<GenericRecord, Row, Schema> getGenericTestData() {
    final String schemaString = "{\"type\":\"record\",\"name\":\"GenericUser\",\"namespace\":\"org.apache.flink.formats.avro.generated\"," + "\"fields\": [{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]}," + "{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]},{\"name\":\"type_long_test\",\"type\":[\"long\",\"null\"]}" + ",{\"name\":\"type_double_test\",\"type\":\"double\"},{\"name\":\"type_null_test\",\"type\":[\"null\"]}," + "{\"name\":\"type_bool_test\",\"type\":[\"boolean\"]},{\"name\":\"type_array_string\",\"type\":" + "{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"type_array_boolean\",\"type\":{\"type\":\"array\"," + "\"items\":\"boolean\"}},{\"name\":\"type_nullable_array\",\"type\":[\"null\",{\"type\":\"array\"," + "\"items\":\"string\"}],\"default\":null},{\"name\":\"type_enum\",\"type\":{\"type\":\"enum\"," + "\"name\":\"Colors\",\"symbols\":[\"RED\",\"GREEN\",\"BLUE\"]}},{\"name\":\"type_map\",\"type\":{\"type\":\"map\"," + "\"values\":\"long\"}},{\"name\":\"type_fixed\",\"type\":[\"null\",{\"type\":\"fixed\",\"name\":\"Fixed16\"," + "\"size\":16}],\"size\":16},{\"name\":\"type_union\",\"type\":[\"null\",\"boolean\",\"long\",\"double\"]}," + "{\"name\":\"type_nested\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Address\",\"fields\":[{\"name\":\"num\"," + "\"type\":\"int\"},{\"name\":\"street\",\"type\":\"string\"},{\"name\":\"city\",\"type\":\"string\"}," + "{\"name\":\"state\",\"type\":\"string\"},{\"name\":\"zip\",\"type\":\"string\"}]}]},{\"name\":\"type_bytes\"," + "\"type\":\"bytes\"},{\"name\":\"type_date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}," + "{\"name\":\"type_time_millis\",\"type\":{\"type\":\"int\",\"logicalType\":\"time-millis\"}},{\"name\":\"type_time_micros\"," + "\"type\":{\"type\":\"long\",\"logicalType\":\"time-micros\"}},{\"name\":\"type_timestamp_millis\",\"type\":{\"type\":\"long\"," + "\"logicalType\":\"timestamp-millis\"}},{\"name\":\"type_timestamp_micros\",\"type\":{\"type\":\"long\"," + "\"logicalType\":\"timestamp-micros\"}},{\"name\":\"type_decimal_bytes\",\"type\":{\"type\":\"bytes\"," + "\"logicalType\":\"decimal\",\"precision\":4,\"scale\":2}},{\"name\":\"type_decimal_fixed\",\"type\":{\"type\":\"fixed\"," + "\"name\":\"Fixed2\",\"size\":2,\"logicalType\":\"decimal\",\"precision\":4,\"scale\":2}}]}";
    final Schema schema = new Schema.Parser().parse(schemaString);
    GenericRecord addr = new GenericData.Record(schema.getField("type_nested").schema().getTypes().get(1));
    addr.put("num", 42);
    addr.put("street", "Main Street 42");
    addr.put("city", "Test City");
    addr.put("state", "Test State");
    addr.put("zip", "12345");
    final Row rowAddr = new Row(5);
    rowAddr.setField(0, 42);
    rowAddr.setField(1, "Main Street 42");
    rowAddr.setField(2, "Test City");
    rowAddr.setField(3, "Test State");
    rowAddr.setField(4, "12345");
    final GenericRecord user = new GenericData.Record(schema);
    user.put("name", "Charlie");
    user.put("favorite_number", null);
    user.put("favorite_color", "blue");
    user.put("type_long_test", 1337L);
    user.put("type_double_test", 1.337d);
    user.put("type_null_test", null);
    user.put("type_bool_test", false);
    user.put("type_array_string", Arrays.asList("hello", "world"));
    user.put("type_array_boolean", Arrays.asList(true, true, false));
    user.put("type_nullable_array", null);
    user.put("type_enum", new GenericData.EnumSymbol(schema.getField("type_enum").schema(), "RED"));
    user.put("type_map", Collections.singletonMap("test", 12L));
    user.put("type_fixed", new Fixed16(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }));
    user.put("type_union", 12.0);
    user.put("type_nested", addr);
    user.put("type_bytes", ByteBuffer.allocate(10));
    user.put("type_date", LocalDate.parse("2014-03-01"));
    user.put("type_time_millis", LocalTime.parse("12:12:12"));
    user.put("type_time_micros", LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS));
    user.put("type_timestamp_millis", Instant.parse("2014-03-01T12:12:12.321Z"));
    user.put("type_timestamp_micros", Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS));
    user.put("type_decimal_bytes", ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    user.put("type_decimal_fixed", new GenericData.Fixed(schema.getField("type_decimal_fixed").schema(), BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    final Row rowUser = new Row(23);
    rowUser.setField(0, "Charlie");
    rowUser.setField(1, null);
    rowUser.setField(2, "blue");
    rowUser.setField(3, 1337L);
    rowUser.setField(4, 1.337d);
    rowUser.setField(5, null);
    rowUser.setField(6, false);
    rowUser.setField(7, new String[] { "hello", "world" });
    rowUser.setField(8, new Boolean[] { true, true, false });
    rowUser.setField(9, null);
    rowUser.setField(10, "RED");
    rowUser.setField(11, Collections.singletonMap("test", 12L));
    rowUser.setField(12, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 });
    rowUser.setField(13, 12.0);
    rowUser.setField(14, rowAddr);
    rowUser.setField(15, new byte[10]);
    rowUser.setField(16, Date.valueOf("2014-03-01"));
    rowUser.setField(17, Time.valueOf("12:12:12"));
    rowUser.setField(18, Time.valueOf(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(19, Timestamp.valueOf("2014-03-01 12:12:12.321"));
    rowUser.setField(20, Timestamp.from(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(21, BigDecimal.valueOf(2000, 2));
    rowUser.setField(22, BigDecimal.valueOf(2000, 2));
    final Tuple3<GenericRecord, Row, Schema> t = new Tuple3<>();
    t.f0 = user;
    t.f1 = rowUser;
    t.f2 = schema;
    return t;
}

Example 2 with Fixed16

use of org.apache.flink.formats.avro.generated.Fixed16 in project flink by apache.

the class AvroTypeExtractionTest method testField.

private void testField(final String fieldName) throws Exception {
    before();
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Path in = new Path(inFile.getAbsoluteFile().toURI());
    AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
    DataSet<User> usersDS = env.createInput(users);
    DataSet<Object> res = usersDS.groupBy(fieldName).reduceGroup((GroupReduceFunction<User, Object>) (values, out) -> {
        for (User u : values) {
            out.collect(u.get(fieldName));
        }
    }).returns(Object.class);
    res.writeAsText(resultPath);
    env.execute("Simple Avro read job");
    // test if automatic registration of the Types worked
    ExecutionConfig ec = env.getConfig();
    Assert.assertTrue(ec.getRegisteredKryoTypes().contains(Fixed16.class));
    switch(fieldName) {
        case "name":
            expected = "Alyssa\nCharlie";
            break;
        case "type_enum":
            expected = "GREEN\nRED\n";
            break;
        case "type_double_test":
            expected = "123.45\n1.337\n";
            break;
        default:
            Assert.fail("Unknown field");
            break;
    }
    after();
}

Example 3 with Fixed16

use of org.apache.flink.formats.avro.generated.Fixed16 in project flink by apache.

the class EncoderDecoderTest method testGeneratedObjectWithNullableFields.

@Test
public void testGeneratedObjectWithNullableFields() {
    List<CharSequence> strings = Arrays.asList(new CharSequence[] { "These", "strings", "should", "be", "recognizable", "as", "a", "meaningful", "sequence" });
    List<Boolean> bools = Arrays.asList(true, true, false, false, true, false, true, true);
    Map<CharSequence, Long> map = new HashMap<>();
    map.put("1", 1L);
    map.put("2", 2L);
    map.put("3", 3L);
    byte[] b = new byte[16];
    new Random().nextBytes(b);
    Fixed16 f = new Fixed16(b);
    Address addr = new Address(239, "6th Main", "Bangalore", "Karnataka", "560075");
    User user = new User("Freudenreich", 1337, "macintosh gray", 1234567890L, 3.1415926, null, true, strings, bools, null, Colors.GREEN, map, f, Boolean.TRUE, addr, ByteBuffer.wrap(b), LocalDate.parse("2014-03-01"), LocalTime.parse("12:12:12"), LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS), Instant.parse("2014-03-01T12:12:12.321Z"), Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS), ByteBuffer.wrap(// 20.00
    BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()), new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    testObjectSerialization(user);
}

Example 4 with Fixed16

use of org.apache.flink.formats.avro.generated.Fixed16 in project flink by apache.

the class AvroTestUtils method getSpecificTestData.

/**
 * Tests all Avro data types as well as nested types for a specific record.
 */
public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getSpecificTestData() {
    final Address addr = Address.newBuilder().setNum(42).setStreet("Main Street 42").setCity("Test City").setState("Test State").setZip("12345").build();
    final Row rowAddr = new Row(5);
    rowAddr.setField(0, 42);
    rowAddr.setField(1, "Main Street 42");
    rowAddr.setField(2, "Test City");
    rowAddr.setField(3, "Test State");
    rowAddr.setField(4, "12345");
    final User user = User.newBuilder().setName("Charlie").setFavoriteNumber(null).setFavoriteColor("blue").setTypeLongTest(1337L).setTypeDoubleTest(1.337d).setTypeNullTest(null).setTypeBoolTest(false).setTypeArrayString(Arrays.asList("hello", "world")).setTypeArrayBoolean(Arrays.asList(true, true, false)).setTypeNullableArray(null).setTypeEnum(Colors.RED).setTypeMap(Collections.singletonMap("test", 12L)).setTypeFixed(new Fixed16(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 })).setTypeUnion(12.0).setTypeNested(addr).setTypeBytes(ByteBuffer.allocate(10)).setTypeDate(LocalDate.parse("2014-03-01")).setTypeTimeMillis(LocalTime.parse("12:12:12")).setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)).setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z")).setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)).setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).build();
    final Row rowUser = new Row(23);
    rowUser.setField(0, "Charlie");
    rowUser.setField(1, null);
    rowUser.setField(2, "blue");
    rowUser.setField(3, 1337L);
    rowUser.setField(4, 1.337d);
    rowUser.setField(5, null);
    rowUser.setField(6, false);
    rowUser.setField(7, new String[] { "hello", "world" });
    rowUser.setField(8, new Boolean[] { true, true, false });
    rowUser.setField(9, null);
    rowUser.setField(10, "RED");
    rowUser.setField(11, Collections.singletonMap("test", 12L));
    rowUser.setField(12, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 });
    rowUser.setField(13, 12.0);
    rowUser.setField(14, rowAddr);
    rowUser.setField(15, new byte[10]);
    rowUser.setField(16, Date.valueOf("2014-03-01"));
    rowUser.setField(17, Time.valueOf("12:12:12"));
    rowUser.setField(18, Time.valueOf(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(19, Timestamp.valueOf("2014-03-01 12:12:12.321"));
    rowUser.setField(20, Timestamp.from(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)));
    rowUser.setField(21, BigDecimal.valueOf(2000, 2));
    rowUser.setField(22, BigDecimal.valueOf(2000, 2));
    final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
    t.f0 = User.class;
    t.f1 = user;
    t.f2 = rowUser;
    return t;
}

Example 5 with Fixed16

use of org.apache.flink.formats.avro.generated.Fixed16 in project flink by apache.

the class AvroSplittableInputFormatTest method createFiles.

@Before
public void createFiles() throws IOException {
    testFile = File.createTempFile("AvroSplittableInputFormatTest", null);
    ArrayList<CharSequence> stringArray = new ArrayList<>();
    stringArray.add(TEST_ARRAY_STRING_1);
    stringArray.add(TEST_ARRAY_STRING_2);
    ArrayList<Boolean> booleanArray = new ArrayList<>();
    booleanArray.add(TEST_ARRAY_BOOLEAN_1);
    booleanArray.add(TEST_ARRAY_BOOLEAN_2);
    HashMap<CharSequence, Long> longMap = new HashMap<>();
    longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
    longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
    Address addr = new Address();
    addr.setNum(TEST_NUM);
    addr.setStreet(TEST_STREET);
    addr.setCity(TEST_CITY);
    addr.setState(TEST_STATE);
    addr.setZip(TEST_ZIP);
    User user1 = new User();
    user1.setName(TEST_NAME);
    user1.setFavoriteNumber(256);
    user1.setTypeDoubleTest(123.45d);
    user1.setTypeBoolTest(true);
    user1.setTypeArrayString(stringArray);
    user1.setTypeArrayBoolean(booleanArray);
    user1.setTypeEnum(TEST_ENUM_COLOR);
    user1.setTypeMap(longMap);
    user1.setTypeNested(addr);
    user1.setTypeBytes(ByteBuffer.allocate(10));
    user1.setTypeDate(LocalDate.parse("2014-03-01"));
    user1.setTypeTimeMillis(LocalTime.parse("12:12:12"));
    user1.setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS));
    user1.setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z"));
    user1.setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS));
    // 20.00
    user1.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    // 20.00
    user1.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
    // Construct via builder
    User user2 = User.newBuilder().setName(TEST_NAME).setFavoriteColor("blue").setFavoriteNumber(null).setTypeBoolTest(false).setTypeDoubleTest(1.337d).setTypeNullTest(null).setTypeLongTest(1337L).setTypeArrayString(new ArrayList<>()).setTypeArrayBoolean(new ArrayList<>()).setTypeNullableArray(null).setTypeEnum(Colors.RED).setTypeMap(new HashMap<>()).setTypeFixed(new Fixed16()).setTypeUnion(123L).setTypeNested(Address.newBuilder().setNum(TEST_NUM).setStreet(TEST_STREET).setCity(TEST_CITY).setState(TEST_STATE).setZip(TEST_ZIP).build()).setTypeBytes(ByteBuffer.allocate(10)).setTypeDate(LocalDate.parse("2014-03-01")).setTypeTimeMillis(LocalTime.parse("12:12:12")).setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS)).setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z")).setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS)).setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())).build();
    DatumWriter<User> userDatumWriter = new SpecificDatumWriter<>(User.class);
    DataFileWriter<User> dataFileWriter = new DataFileWriter<>(userDatumWriter);
    dataFileWriter.create(user1.getSchema(), testFile);
    dataFileWriter.append(user1);
    dataFileWriter.append(user2);
    Random rnd = new Random(1337);
    for (int i = 0; i < NUM_RECORDS - 2; i++) {
        User user = new User();
        user.setName(TEST_NAME + rnd.nextInt());
        user.setFavoriteNumber(rnd.nextInt());
        user.setTypeDoubleTest(rnd.nextDouble());
        user.setTypeBoolTest(true);
        user.setTypeArrayString(stringArray);
        user.setTypeArrayBoolean(booleanArray);
        user.setTypeEnum(TEST_ENUM_COLOR);
        user.setTypeMap(longMap);
        Address address = new Address();
        address.setNum(TEST_NUM);
        address.setStreet(TEST_STREET);
        address.setCity(TEST_CITY);
        address.setState(TEST_STATE);
        address.setZip(TEST_ZIP);
        user.setTypeNested(address);
        user.setTypeBytes(ByteBuffer.allocate(10));
        user.setTypeDate(LocalDate.parse("2014-03-01"));
        user.setTypeTimeMillis(LocalTime.parse("12:12:12"));
        user.setTypeTimeMicros(LocalTime.ofSecondOfDay(0).plus(123456L, ChronoUnit.MICROS));
        user.setTypeTimestampMillis(Instant.parse("2014-03-01T12:12:12.321Z"));
        user.setTypeTimestampMicros(Instant.ofEpochSecond(0).plus(123456L, ChronoUnit.MICROS));
        // 20.00
        user.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
        // 20.00
        user.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
}