Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 76 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteJsonResult method testMissingFieldInWriteRecord.

@Test
public void testMissingFieldInWriteRecord() throws IOException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false, NullSuppression.NEVER_SUPPRESS, null, null, null)) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
    }
    final byte[] data = baos.toByteArray();
    final String expected = "[{\"id\":\"1\",\"name\":null}]";
    final String output = new String(data, StandardCharsets.UTF_8);
    assertEquals(expected, output);
}

Example 77 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestWriteJsonResult method testMissingAndExtraFieldInWriteRecord.

@Test
public void testMissingAndExtraFieldInWriteRecord() throws IOException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    values.put("dob", "1/1/1970");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false, NullSuppression.NEVER_SUPPRESS, null, null, null)) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
    }
    final byte[] data = baos.toByteArray();
    final String expected = "[{\"id\":\"1\",\"name\":null}]";
    final String output = new String(data, StandardCharsets.UTF_8);
    assertEquals(expected, output);
}

Example 78 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestLookupRecord method testAddFieldsToExistingRecord.

@Test
public void testAddFieldsToExistingRecord() throws InitializationException, IOException {
    final RecordLookup lookupService = new RecordLookup();
    runner.addControllerService("lookup", lookupService);
    runner.enableControllerService(lookupService);
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("favorite", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("least", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Record sports = new MapRecord(schema, new HashMap<String, Object>());
    sports.setValue("favorite", "basketball");
    sports.setValue("least", "soccer");
    lookupService.addValue("John Doe", sports);
    recordReader = new MockRecordParser();
    recordReader.addSchemaField("name", RecordFieldType.STRING);
    recordReader.addSchemaField("age", RecordFieldType.INT);
    recordReader.addSchemaField("favorite", RecordFieldType.STRING);
    recordReader.addSchemaField("least", RecordFieldType.STRING);
    recordReader.addRecord("John Doe", 48, null, "baseball");
    runner.addControllerService("reader", recordReader);
    runner.enableControllerService(recordReader);
    runner.setProperty("lookup", "/name");
    runner.setProperty(LookupRecord.RESULT_RECORD_PATH, "/");
    runner.setProperty(LookupRecord.RESULT_CONTENTS, LookupRecord.RESULT_RECORD_FIELDS);
    runner.enqueue("");
    runner.run();
    final MockFlowFile out = runner.getFlowFilesForRelationship(LookupRecord.REL_MATCHED).get(0);
    out.assertContentEquals("John Doe,48,basketball,soccer\n");
}

Example 79 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class TestLookupRecord method testAddFieldsToNonExistentRecord.

/**
 * If the output fields are added to a record that doesn't exist, the result should be that a Record is
 * created and the results added to it.
 */
@Test
public void testAddFieldsToNonExistentRecord() throws InitializationException {
    final RecordLookup lookupService = new RecordLookup();
    runner.addControllerService("lookup", lookupService);
    runner.enableControllerService(lookupService);
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("favorite", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("least", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Record sports = new MapRecord(schema, new HashMap<String, Object>());
    sports.setValue("favorite", "basketball");
    sports.setValue("least", "soccer");
    lookupService.addValue("John Doe", sports);
    recordReader = new MockRecordParser();
    recordReader.addSchemaField("name", RecordFieldType.STRING);
    recordReader.addSchemaField("age", RecordFieldType.INT);
    recordReader.addSchemaField("sport", RecordFieldType.RECORD);
    recordReader.addRecord("John Doe", 48, null);
    runner.addControllerService("reader", recordReader);
    runner.enableControllerService(recordReader);
    runner.setProperty("lookup", "/name");
    runner.setProperty(LookupRecord.RESULT_RECORD_PATH, "/sport");
    runner.setProperty(LookupRecord.RESULT_CONTENTS, LookupRecord.RESULT_RECORD_FIELDS);
    runner.enqueue("");
    runner.run();
    final MockFlowFile out = runner.getFlowFilesForRelationship(LookupRecord.REL_MATCHED).get(0);
    // We can't be sure of the order of the fields in the record, so we allow either 'least' or 'favorite' to be first
    final String outputContents = new String(out.toByteArray());
    assertTrue(outputContents.equals("John Doe,48,MapRecord[{favorite=basketball, least=soccer}]\n") || outputContents.equals("John Doe,48,MapRecord[{least=soccer, favorite=basketball}]\n"));
}

Example 80 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class DataTypeUtils method convertRecordFieldtoObject.

/**
 * Creates a native Java object from a given object of a specified type. Non-scalar (complex, nested, etc.) data types are processed iteratively/recursively, such that all
 * included objects are native Java objects, rather than Record API objects or implementation-specific objects.
 * @param value The object to be converted
 * @param dataType The type of the provided object
 * @return An object representing a native Java conversion of the given input object
 */
public static Object convertRecordFieldtoObject(final Object value, final DataType dataType) {
    if (value == null) {
        return null;
    }
    if (value instanceof Record) {
        Record record = (Record) value;
        RecordSchema recordSchema = record.getSchema();
        if (recordSchema == null) {
            throw new IllegalTypeConversionException("Cannot convert value of type Record to Map because Record does not have an associated Schema");
        }
        final Map<String, Object> recordMap = new HashMap<>();
        for (RecordField field : recordSchema.getFields()) {
            final DataType fieldDataType = field.getDataType();
            final String fieldName = field.getFieldName();
            Object fieldValue = record.getValue(fieldName);
            if (fieldValue == null) {
                recordMap.put(fieldName, null);
            } else if (isScalarValue(fieldDataType, fieldValue)) {
                recordMap.put(fieldName, fieldValue);
            } else if (fieldDataType instanceof RecordDataType) {
                Record nestedRecord = (Record) fieldValue;
                recordMap.put(fieldName, convertRecordFieldtoObject(nestedRecord, fieldDataType));
            } else if (fieldDataType instanceof MapDataType) {
                recordMap.put(fieldName, convertRecordMapToJavaMap((Map) fieldValue, ((MapDataType) fieldDataType).getValueType()));
            } else if (fieldDataType instanceof ArrayDataType) {
                recordMap.put(fieldName, convertRecordArrayToJavaArray((Object[]) fieldValue, ((ArrayDataType) fieldDataType).getElementType()));
            } else {
                throw new IllegalTypeConversionException("Cannot convert value [" + fieldValue + "] of type " + fieldDataType.toString() + " to Map for field " + fieldName + " because the type is not supported");
            }
        }
        return recordMap;
    } else if (value instanceof Map) {
        return convertRecordMapToJavaMap((Map) value, ((MapDataType) dataType).getValueType());
    } else if (dataType != null && isScalarValue(dataType, value)) {
        return value;
    }
    throw new IllegalTypeConversionException("Cannot convert value of class " + value.getClass().getName() + " because the type is not supported");
}