Examples with SimpleRecordSchema org.apache.nifi.serialization.SimpleRecordSchema used on opensource projects

Example 86 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestRecordPath method testWildcardChild.

@Test
public void testWildcardChild() {
    final Map<String, Object> accountValues = new HashMap<>();
    accountValues.put("id", 1);
    accountValues.put("balance", 123.45D);
    final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
    final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
    final Map<String, Object> values = new HashMap<>();
    values.put("id", 48);
    values.put("name", "John Doe");
    values.put("mainAccount", accountRecord);
    final Record record = new MapRecord(schema, values);
    final List<FieldValue> fieldValues = RecordPath.compile("/mainAccount/*").evaluate(record).getSelectedFields().collect(Collectors.toList());
    assertEquals(2, fieldValues.size());
    for (final FieldValue fieldValue : fieldValues) {
        assertEquals(accountRecord, fieldValue.getParentRecord().get());
    }
    assertEquals("id", fieldValues.get(0).getField().getFieldName());
    assertEquals(1, fieldValues.get(0).getValue());
    assertEquals("balance", fieldValues.get(1).getField().getFieldName());
    assertEquals(123.45D, fieldValues.get(1).getValue());
    RecordPath.compile("/mainAccount/*[. > 100]").evaluate(record).getSelectedFields().forEach(field -> field.updateValue(122.44D));
    assertEquals(1, accountValues.get("id"));
    assertEquals(122.44D, accountValues.get("balance"));
}

Example 87 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class AvroTypeUtil method createSchema.

/**
 * Converts an Avro Schema to a RecordSchema
 *
 * @param avroSchema the Avro Schema to convert
 * @param schemaText the textual representation of the schema
 * @param schemaId the identifier of the schema
 * @return the Corresponding Record Schema
 */
public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) {
    if (avroSchema == null) {
        throw new IllegalArgumentException("Avro Schema cannot be null");
    }
    String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName();
    SimpleRecordSchema recordSchema = new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId);
    DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema);
    Map<String, DataType> knownRecords = new HashMap<>();
    knownRecords.put(schemaFullName, recordSchemaType);
    final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size());
    for (final Field field : avroSchema.getFields()) {
        final String fieldName = field.name();
        final Schema fieldSchema = field.schema();
        final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords);
        final boolean nullable = isNullable(fieldSchema);
        addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable);
    }
    recordSchema.setFields(recordFields);
    return recordSchema;
}

Example 88 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestAvroTypeUtil method testCreateAvroSchemaPrimitiveTypes.

@Test
public void testCreateAvroSchemaPrimitiveTypes() throws SchemaNotFoundException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("int", RecordFieldType.INT.getDataType()));
    fields.add(new RecordField("long", RecordFieldType.LONG.getDataType()));
    fields.add(new RecordField("string", RecordFieldType.STRING.getDataType(), "hola", Collections.singleton("greeting")));
    fields.add(new RecordField("byte", RecordFieldType.BYTE.getDataType()));
    fields.add(new RecordField("char", RecordFieldType.CHAR.getDataType()));
    fields.add(new RecordField("short", RecordFieldType.SHORT.getDataType()));
    fields.add(new RecordField("double", RecordFieldType.DOUBLE.getDataType()));
    fields.add(new RecordField("float", RecordFieldType.FLOAT.getDataType()));
    fields.add(new RecordField("time", RecordFieldType.TIME.getDataType()));
    fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
    fields.add(new RecordField("timestamp", RecordFieldType.TIMESTAMP.getDataType()));
    final DataType arrayType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType());
    fields.add(new RecordField("strings", arrayType));
    final DataType mapType = RecordFieldType.MAP.getMapDataType(RecordFieldType.LONG.getDataType());
    fields.add(new RecordField("map", mapType));
    final List<RecordField> personFields = new ArrayList<>();
    personFields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    personFields.add(new RecordField("dob", RecordFieldType.DATE.getDataType()));
    final RecordSchema personSchema = new SimpleRecordSchema(personFields);
    final DataType personType = RecordFieldType.RECORD.getRecordDataType(personSchema);
    fields.add(new RecordField("person", personType));
    final RecordSchema recordSchema = new SimpleRecordSchema(fields);
    final Schema avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
    // everything should be a union, since it's nullable.
    for (final Field field : avroSchema.getFields()) {
        final Schema fieldSchema = field.schema();
        assertEquals(Type.UNION, fieldSchema.getType());
        assertTrue("Field " + field.name() + " does not contain NULL type", fieldSchema.getTypes().contains(Schema.create(Type.NULL)));
    }
    final RecordSchema afterConversion = AvroTypeUtil.createSchema(avroSchema);
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("int").get());
    assertEquals(RecordFieldType.LONG.getDataType(), afterConversion.getDataType("long").get());
    assertEquals(RecordFieldType.STRING.getDataType(), afterConversion.getDataType("string").get());
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("byte").get());
    assertEquals(RecordFieldType.STRING.getDataType(), afterConversion.getDataType("char").get());
    assertEquals(RecordFieldType.INT.getDataType(), afterConversion.getDataType("short").get());
    assertEquals(RecordFieldType.DOUBLE.getDataType(), afterConversion.getDataType("double").get());
    assertEquals(RecordFieldType.FLOAT.getDataType(), afterConversion.getDataType("float").get());
    assertEquals(RecordFieldType.TIME.getDataType(), afterConversion.getDataType("time").get());
    assertEquals(RecordFieldType.DATE.getDataType(), afterConversion.getDataType("date").get());
    assertEquals(RecordFieldType.TIMESTAMP.getDataType(), afterConversion.getDataType("timestamp").get());
    assertEquals(arrayType, afterConversion.getDataType("strings").get());
    assertEquals(mapType, afterConversion.getDataType("map").get());
    assertEquals(personType, afterConversion.getDataType("person").get());
    final RecordField stringField = afterConversion.getField("string").get();
    assertEquals("hola", stringField.getDefaultValue());
    assertEquals(Collections.singleton("greeting"), stringField.getAliases());
}

Example 89 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class MockRecordParser method createRecordReader.

@Override
public RecordReader createRecordReader(Map<String, String> variables, InputStream in, ComponentLog logger) throws IOException, SchemaNotFoundException {
    final Iterator<Object[]> itr = records.iterator();
    return new RecordReader() {

        private int recordCount = 0;

        @Override
        public void close() throws IOException {
        }

        @Override
        public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException {
            if (failAfterN >= recordCount) {
                throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read");
            }
            recordCount++;
            if (!itr.hasNext()) {
                return null;
            }
            final Object[] values = itr.next();
            final Map<String, Object> valueMap = new HashMap<>();
            int i = 0;
            for (final RecordField field : fields) {
                final String fieldName = field.getFieldName();
                valueMap.put(fieldName, values[i++]);
            }
            return new MapRecord(new SimpleRecordSchema(fields), valueMap);
        }

        @Override
        public RecordSchema getSchema() {
            return new SimpleRecordSchema(fields);
        }
    };
}

Example 90 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestConfluentSchemaRegistryWriter method createRecordSchema.

private RecordSchema createRecordSchema(final SchemaIdentifier schemaIdentifier) {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("firstName", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("lastName", RecordFieldType.STRING.getDataType()));
    return new SimpleRecordSchema(fields, schemaIdentifier);
}