Examples with SamzaSqlRelRecord org.apache.samza.sql.SamzaSqlRelRecord used on opensource projects

Example 1 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class AvroRelConverter method convertToGenericRecord.

private static GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) {
    GenericRecord record = new GenericData.Record(schema);
    List<String> fieldNames = relRecord.getFieldNames();
    List<Object> values = relRecord.getFieldValues();
    for (int index = 0; index < fieldNames.size(); index++) {
        if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) {
            String fieldName = fieldNames.get(index);
            /**
             * It is possible that the destination Avro schema doesn't have all the fields that are projected from the
             * SQL. This is especially possible in SQL statements like
             *        insert into kafka.outputTopic select id, company from profile
             * where company is an avro record in itself whose schema can evolve. When this happens we will end up with
             * fields in the SamzaSQLRelRecord for company field which doesn't have equivalent fields in the outputTopic's schema
             * for company. To support this scenario where the input schemas and output schemas can evolve in their own cadence,
             * We ignore the fields which doesn't have corresponding schema in the output topic.
             */
            if (schema.getField(fieldName) == null) {
                LOG.debug("Schema with Name {} and Namespace {} doesn't contain the fieldName {}, Skipping it.", schema.getName(), schema.getNamespace(), fieldName);
                continue;
            }
            Object relObj = values.get(index);
            Schema fieldSchema = schema.getField(fieldName).schema();
            record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema)));
        }
    }
    return record;
}

Example 2 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class AvroRelConverter method convertToRelRecord.

private static SamzaSqlRelRecord convertToRelRecord(IndexedRecord avroRecord) {
    List<Object> fieldValues = new ArrayList<>();
    List<String> fieldNames = new ArrayList<>();
    if (avroRecord != null) {
        fieldNames.addAll(avroRecord.getSchema().getFields().stream().map(Schema.Field::name).collect(Collectors.toList()));
        fieldValues.addAll(avroRecord.getSchema().getFields().stream().map(f -> convertToJavaObject(avroRecord.get(avroRecord.getSchema().getField(f.name()).pos()), getNonNullUnionSchema(avroRecord.getSchema().getField(f.name()).schema()))).collect(Collectors.toList()));
    } else {
        String msg = "Avro Record is null";
        LOG.error(msg);
        throw new SamzaException(msg);
    }
    return new SamzaSqlRelRecord(fieldNames, fieldValues);
}

Example 3 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class GetSqlFieldUdf method extractField.

static Object extractField(String fieldName, Object current, boolean validateField) {
    if (current instanceof SamzaSqlRelRecord) {
        SamzaSqlRelRecord record = (SamzaSqlRelRecord) current;
        if (validateField) {
            Validate.isTrue(record.getFieldNames().contains(fieldName), String.format("Invalid field %s in record %s", fieldName, record));
        }
        return record.getField(fieldName).orElse(null);
    } else if (current instanceof Map) {
        Map map = (Map) current;
        if (map.containsKey(fieldName)) {
            return map.get(fieldName);
        } else if (map.containsKey(new Utf8(fieldName))) {
            return map.get(new Utf8(fieldName));
        } else {
            throw new IllegalArgumentException(String.format("Couldn't find the field %s in map %s", fieldName, map));
        }
    } else if (current instanceof List && fieldName.endsWith("]")) {
        List list = (List) current;
        int index = Integer.parseInt(fieldName.substring(fieldName.indexOf("[") + 1, fieldName.length() - 1));
        return list.get(index);
    }
    throw new IllegalArgumentException(String.format("Unsupported accessing operation for data type: %s with field: %s.", current.getClass(), fieldName));
}

Example 4 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestSamzaSqlRelMessage method testCompositeKeyCreationWithInEqualKeyNameValues.

@Test(expected = IllegalArgumentException.class)
public void testCompositeKeyCreationWithInEqualKeyNameValues() {
    List<String> keyPartNames = Arrays.asList("kfield1", "kfield2");
    SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata(0L, 0L));
    SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1)));
}

Example 5 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestSamzaSqlRelRecordSerde method testNestedRecordConversion.

@Test
public void testNestedRecordConversion() {
    Map<String, String> props = new HashMap<>();
    SystemStream ss1 = new SystemStream("test", "nestedRecord");
    props.put(String.format(ConfigBasedAvroRelSchemaProviderFactory.CFG_SOURCE_SCHEMA, ss1.getSystem(), ss1.getStream()), Profile.SCHEMA$.toString());
    ConfigBasedAvroRelSchemaProviderFactory factory = new ConfigBasedAvroRelSchemaProviderFactory();
    AvroRelSchemaProvider nestedRecordSchemaProvider = (AvroRelSchemaProvider) factory.create(ss1, new MapConfig(props));
    AvroRelConverter nestedRecordAvroRelConverter = new AvroRelConverter(ss1, nestedRecordSchemaProvider, new MapConfig());
    Pair<SamzaSqlRelMessage, GenericData.Record> messageRecordPair = TestSamzaSqlRelMessageSerde.createNestedSamzaSqlRelMessage(nestedRecordAvroRelConverter);
    SamzaSqlRelRecordSerdeFactory.SamzaSqlRelRecordSerde serde = (SamzaSqlRelRecordSerdeFactory.SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null);
    SamzaSqlRelRecord resultRecord = serde.fromBytes(serde.toBytes(messageRecordPair.getKey().getSamzaSqlRelRecord()));
    GenericData.Record recordPostConversion = (GenericData.Record) nestedRecordAvroRelConverter.convertToAvroObject(resultRecord, Profile.SCHEMA$);
    for (Schema.Field field : Profile.SCHEMA$.getFields()) {
        // equals() on GenericRecord does the nested record equality check as well.
        Assert.assertEquals(messageRecordPair.getValue().get(field.name()), recordPostConversion.get(field.name()));
    }
}