Examples with SamzaSqlRelMessage org.apache.samza.sql.data.SamzaSqlRelMessage used on opensource projects

Example 26 with SamzaSqlRelMessage

use of org.apache.samza.sql.data.SamzaSqlRelMessage in project samza by apache.

the class TestAvroRelConversion method testNestedRecordConversion.

@Test
public void testNestedRecordConversion() throws IOException {
    GenericData.Record record = new GenericData.Record(Profile.SCHEMA$);
    record.put("id", 1);
    record.put("name", "name1");
    record.put("companyId", 0);
    GenericData.Record addressRecord = new GenericData.Record(AddressRecord.SCHEMA$);
    addressRecord.put("zip", 90000);
    GenericData.Record streetNumRecord = new GenericData.Record(StreetNumRecord.SCHEMA$);
    streetNumRecord.put("number", 1200);
    addressRecord.put("streetnum", streetNumRecord);
    record.put("address", addressRecord);
    record.put("selfEmployed", "True");
    GenericData.Record phoneNumberRecordH = new GenericData.Record(PhoneNumber.SCHEMA$);
    phoneNumberRecordH.put("kind", Kind.Home);
    phoneNumberRecordH.put("number", "111-111-1111");
    GenericData.Record phoneNumberRecordC = new GenericData.Record(PhoneNumber.SCHEMA$);
    phoneNumberRecordC.put("kind", Kind.Cell);
    phoneNumberRecordC.put("number", "111-111-1112");
    List<GenericData.Record> phoneNumbers = new ArrayList<>();
    phoneNumbers.add(phoneNumberRecordH);
    phoneNumbers.add(phoneNumberRecordC);
    record.put("phoneNumbers", phoneNumbers);
    GenericData.Record simpleRecord1 = new GenericData.Record(SimpleRecord.SCHEMA$);
    simpleRecord1.put("id", 1);
    simpleRecord1.put("name", "name1");
    GenericData.Record simpleRecord2 = new GenericData.Record(SimpleRecord.SCHEMA$);
    simpleRecord2.put("id", 2);
    simpleRecord2.put("name", "name2");
    HashMap<String, IndexedRecord> mapValues = new HashMap<>();
    mapValues.put("key1", simpleRecord1);
    mapValues.put("key2", simpleRecord2);
    record.put("mapValues", mapValues);
    SamzaSqlRelMessage relMessage = nestedRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record));
    LOG.info(relMessage.toString());
    KV<Object, Object> samzaMessage = nestedRecordAvroRelConverter.convertToSamzaMessage(relMessage);
    GenericRecord recordPostConversion = (GenericRecord) samzaMessage.getValue();
    for (Schema.Field field : Profile.SCHEMA$.getFields()) {
        // equals() on GenericRecord does the nested record equality check as well.
        Assert.assertEquals(record.get(field.name()), recordPostConversion.get(field.name()));
    }
}

Example 27 with SamzaSqlRelMessage

use of org.apache.samza.sql.data.SamzaSqlRelMessage in project samza by apache.

the class TestAvroRelConversion method validateAvroSerializedData.

private void validateAvroSerializedData(byte[] serializedData, Object unionValue) throws IOException {
    GenericRecord complexRecordValue = genericRecordFromBytes(serializedData, ComplexRecord.SCHEMA$);
    SamzaSqlRelMessage message = complexRecordAvroRelConverter.convertToRelMessage(new KV<>("key", complexRecordValue));
    Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), ComplexRecord.SCHEMA$.getFields().size() + 1);
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("id").get(), id);
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("bool_value").get(), boolValue);
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("double_value").get(), doubleValue);
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("string_value").get(), new Utf8(testStrValue));
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("float_value0").get(), floatValue);
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("long_value").get(), longValue);
    if (unionValue instanceof String) {
        Assert.assertEquals(message.getSamzaSqlRelRecord().getField("union_value").get(), new Utf8((String) unionValue));
    } else {
        Assert.assertEquals(message.getSamzaSqlRelRecord().getField("union_value").get(), unionValue);
    }
    Assert.assertTrue(arrayValue.stream().map(Utf8::new).collect(Collectors.toList()).equals(message.getSamzaSqlRelRecord().getField("array_values").get()));
    Assert.assertTrue(mapValue.entrySet().stream().collect(Collectors.toMap(x -> new Utf8(x.getKey()), y -> new Utf8(y.getValue()))).equals(message.getSamzaSqlRelRecord().getField("map_values").get()));
    Assert.assertTrue(Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("bytes_value").get()).getBytes(), testBytes.array()));
    Assert.assertTrue(Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("fixed_value").get()).getBytes(), DEFAULT_TRACKING_ID_BYTES));
    LOG.info(message.toString());
    KV<Object, Object> samzaMessage = complexRecordAvroRelConverter.convertToSamzaMessage(message);
    GenericRecord record = (GenericRecord) samzaMessage.getValue();
    for (Schema.Field field : ComplexRecord.SCHEMA$.getFields()) {
        if (field.name().equals("array_values")) {
            Assert.assertTrue(record.get(field.name()).equals(complexRecordValue.get(field.name())));
        } else {
            Object expected = complexRecordValue.get(field.name());
            Assert.assertEquals(expected, record.get(field.name()));
        }
    }
}

Example 28 with SamzaSqlRelMessage

use of org.apache.samza.sql.data.SamzaSqlRelMessage in project samza by apache.

the class AvroRelConverter method convertToRelMessage.

/**
 * Create a SamzaSqlRelMessage for the specified key and Avro record using the schema from the Avro record.
 */
public static SamzaSqlRelMessage convertToRelMessage(Object key, IndexedRecord record, Schema schema) {
    List<String> payloadFieldNames = new ArrayList<>();
    List<Object> payloadFieldValues = new ArrayList<>();
    fetchFieldNamesAndValuesFromIndexedRecord(record, payloadFieldNames, payloadFieldValues, schema);
    return new SamzaSqlRelMessage(key, payloadFieldNames, payloadFieldValues, new SamzaSqlRelMsgMetadata(0L, 0L));
}