Examples with RecordField org.apache.nifi.serialization.record.RecordField used on opensource projects

Example 56 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestWriteJsonResult method testMissingAndExtraFieldInWriteRecord.

@Test
public void testMissingAndExtraFieldInWriteRecord() throws IOException {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    values.put("dob", "1/1/1970");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false, NullSuppression.NEVER_SUPPRESS, null, null, null)) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
    }
    final byte[] data = baos.toByteArray();
    final String expected = "[{\"id\":\"1\",\"name\":null}]";
    final String output = new String(data, StandardCharsets.UTF_8);
    assertEquals(expected, output);
}

Example 57 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestLookupRecord method testAddFieldsToExistingRecord.

@Test
public void testAddFieldsToExistingRecord() throws InitializationException, IOException {
    final RecordLookup lookupService = new RecordLookup();
    runner.addControllerService("lookup", lookupService);
    runner.enableControllerService(lookupService);
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("favorite", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("least", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Record sports = new MapRecord(schema, new HashMap<String, Object>());
    sports.setValue("favorite", "basketball");
    sports.setValue("least", "soccer");
    lookupService.addValue("John Doe", sports);
    recordReader = new MockRecordParser();
    recordReader.addSchemaField("name", RecordFieldType.STRING);
    recordReader.addSchemaField("age", RecordFieldType.INT);
    recordReader.addSchemaField("favorite", RecordFieldType.STRING);
    recordReader.addSchemaField("least", RecordFieldType.STRING);
    recordReader.addRecord("John Doe", 48, null, "baseball");
    runner.addControllerService("reader", recordReader);
    runner.enableControllerService(recordReader);
    runner.setProperty("lookup", "/name");
    runner.setProperty(LookupRecord.RESULT_RECORD_PATH, "/");
    runner.setProperty(LookupRecord.RESULT_CONTENTS, LookupRecord.RESULT_RECORD_FIELDS);
    runner.enqueue("");
    runner.run();
    final MockFlowFile out = runner.getFlowFilesForRelationship(LookupRecord.REL_MATCHED).get(0);
    out.assertContentEquals("John Doe,48,basketball,soccer\n");
}

Example 58 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestLookupRecord method testAddFieldsToNonExistentRecord.

/**
 * If the output fields are added to a record that doesn't exist, the result should be that a Record is
 * created and the results added to it.
 */
@Test
public void testAddFieldsToNonExistentRecord() throws InitializationException {
    final RecordLookup lookupService = new RecordLookup();
    runner.addControllerService("lookup", lookupService);
    runner.enableControllerService(lookupService);
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("favorite", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("least", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Record sports = new MapRecord(schema, new HashMap<String, Object>());
    sports.setValue("favorite", "basketball");
    sports.setValue("least", "soccer");
    lookupService.addValue("John Doe", sports);
    recordReader = new MockRecordParser();
    recordReader.addSchemaField("name", RecordFieldType.STRING);
    recordReader.addSchemaField("age", RecordFieldType.INT);
    recordReader.addSchemaField("sport", RecordFieldType.RECORD);
    recordReader.addRecord("John Doe", 48, null);
    runner.addControllerService("reader", recordReader);
    runner.enableControllerService(recordReader);
    runner.setProperty("lookup", "/name");
    runner.setProperty(LookupRecord.RESULT_RECORD_PATH, "/sport");
    runner.setProperty(LookupRecord.RESULT_CONTENTS, LookupRecord.RESULT_RECORD_FIELDS);
    runner.enqueue("");
    runner.run();
    final MockFlowFile out = runner.getFlowFilesForRelationship(LookupRecord.REL_MATCHED).get(0);
    // We can't be sure of the order of the fields in the record, so we allow either 'least' or 'favorite' to be first
    final String outputContents = new String(out.toByteArray());
    assertTrue(outputContents.equals("John Doe,48,MapRecord[{favorite=basketball, least=soccer}]\n") || outputContents.equals("John Doe,48,MapRecord[{least=soccer, favorite=basketball}]\n"));
}

Example 59 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class SimpleRecordSchema method setFields.

public void setFields(final List<RecordField> fields) {
    if (this.fields != null) {
        throw new IllegalArgumentException("Fields have already been set.");
    }
    this.fields = Collections.unmodifiableList(new ArrayList<>(fields));
    this.fieldMap = new HashMap<>(fields.size() * 2);
    for (final RecordField field : fields) {
        RecordField previousValue = fieldMap.put(field.getFieldName(), field);
        if (previousValue != null) {
            throw new IllegalArgumentException("Two fields are given with the same name (or alias) of '" + field.getFieldName() + "'");
        }
        for (final String alias : field.getAliases()) {
            previousValue = fieldMap.put(alias, field);
            if (previousValue != null) {
                throw new IllegalArgumentException("Two fields are given with the same name (or alias) of '" + field.getFieldName() + "'");
            }
        }
    }
}

Example 60 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class DataTypeUtils method merge.

public static RecordSchema merge(final RecordSchema thisSchema, final RecordSchema otherSchema) {
    if (thisSchema == null) {
        return otherSchema;
    }
    if (otherSchema == null) {
        return thisSchema;
    }
    if (thisSchema == otherSchema) {
        return thisSchema;
    }
    final List<RecordField> otherFields = otherSchema.getFields();
    if (otherFields.isEmpty()) {
        return thisSchema;
    }
    final List<RecordField> thisFields = thisSchema.getFields();
    if (thisFields.isEmpty()) {
        return otherSchema;
    }
    final Map<String, Integer> fieldIndices = new HashMap<>();
    final List<RecordField> fields = new ArrayList<>();
    for (int i = 0; i < thisFields.size(); i++) {
        final RecordField field = thisFields.get(i);
        final Integer index = Integer.valueOf(i);
        fieldIndices.put(field.getFieldName(), index);
        for (final String alias : field.getAliases()) {
            fieldIndices.put(alias, index);
        }
        fields.add(field);
    }
    for (final RecordField otherField : otherFields) {
        Integer fieldIndex = fieldIndices.get(otherField.getFieldName());
        // if one exists.
        if (fieldIndex == null) {
            for (final String alias : otherField.getAliases()) {
                fieldIndex = fieldIndices.get(alias);
                if (fieldIndex != null) {
                    break;
                }
            }
        }
        // If there is no field with the same name then just add 'otherField'.
        if (fieldIndex == null) {
            fields.add(otherField);
            continue;
        }
        // Merge the two fields, if necessary
        final RecordField thisField = fields.get(fieldIndex);
        if (isMergeRequired(thisField, otherField)) {
            final RecordField mergedField = merge(thisField, otherField);
            fields.set(fieldIndex, mergedField);
        }
    }
    return new SimpleRecordSchema(fields);
}