Examples with SimpleRecordSchema org.apache.nifi.serialization.SimpleRecordSchema used on opensource projects

Example 6 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class MongoDBLookupService method lookup.

@Override
public Optional<Object> lookup(Map<String, Object> coordinates) throws LookupFailureException {
    Map<String, Object> clean = new HashMap<>();
    clean.putAll(coordinates);
    Document query = new Document(clean);
    if (coordinates.size() == 0) {
        throw new LookupFailureException("No keys were configured. Mongo query would return random documents.");
    }
    try {
        Document result = this.findOne(query);
        if (result == null) {
            return Optional.empty();
        } else if (!StringUtils.isEmpty(lookupValueField)) {
            return Optional.ofNullable(result.get(lookupValueField));
        } else {
            final List<RecordField> fields = new ArrayList<>();
            for (String key : result.keySet()) {
                if (key.equals("_id")) {
                    continue;
                }
                fields.add(new RecordField(key, RecordFieldType.STRING.getDataType()));
            }
            final RecordSchema schema = new SimpleRecordSchema(fields);
            return Optional.ofNullable(new MapRecord(schema, result));
        }
    } catch (Exception ex) {
        getLogger().error("Error during lookup {}", new Object[] { query.toJson() }, ex);
        throw new LookupFailureException(ex);
    }
}

Example 7 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class GrokReader method createRecordSchema.

static RecordSchema createRecordSchema(final Grok grok) {
    final List<RecordField> fields = new ArrayList<>();
    String grokExpression = grok.getOriginalGrokPattern();
    populateSchemaFieldNames(grok, grokExpression, fields);
    fields.add(new RecordField(GrokRecordReader.STACK_TRACE_COLUMN_NAME, RecordFieldType.STRING.getDataType(), true));
    fields.add(new RecordField(GrokRecordReader.RAW_MESSAGE_NAME, RecordFieldType.STRING.getDataType(), true));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    return schema;
}

Example 8 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class AbstractJsonRowRecordReader method getRawNodeValue.

protected Object getRawNodeValue(final JsonNode fieldNode, final DataType dataType) throws IOException {
    if (fieldNode == null || fieldNode.isNull()) {
        return null;
    }
    if (fieldNode.isNumber()) {
        return fieldNode.getNumberValue();
    }
    if (fieldNode.isBinary()) {
        return fieldNode.getBinaryValue();
    }
    if (fieldNode.isBoolean()) {
        return fieldNode.getBooleanValue();
    }
    if (fieldNode.isTextual()) {
        return fieldNode.getTextValue();
    }
    if (fieldNode.isArray()) {
        final ArrayNode arrayNode = (ArrayNode) fieldNode;
        final int numElements = arrayNode.size();
        final Object[] arrayElements = new Object[numElements];
        int count = 0;
        final DataType elementDataType;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.ARRAY) {
            final ArrayDataType arrayDataType = (ArrayDataType) dataType;
            elementDataType = arrayDataType.getElementType();
        } else {
            elementDataType = null;
        }
        for (final JsonNode node : arrayNode) {
            final Object value = getRawNodeValue(node, elementDataType);
            arrayElements[count++] = value;
        }
        return arrayElements;
    }
    if (fieldNode.isObject()) {
        RecordSchema childSchema;
        if (dataType != null && RecordFieldType.RECORD == dataType.getFieldType()) {
            final RecordDataType recordDataType = (RecordDataType) dataType;
            childSchema = recordDataType.getChildSchema();
        } else {
            childSchema = null;
        }
        if (childSchema == null) {
            childSchema = new SimpleRecordSchema(Collections.emptyList());
        }
        final Iterator<String> fieldNames = fieldNode.getFieldNames();
        final Map<String, Object> childValues = new HashMap<>();
        while (fieldNames.hasNext()) {
            final String childFieldName = fieldNames.next();
            final Object childValue = getRawNodeValue(fieldNode.get(childFieldName), dataType);
            childValues.put(childFieldName, childValue);
        }
        final MapRecord record = new MapRecord(childSchema, childValues);
        return record;
    }
    return null;
}

Example 9 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class JsonPathRowRecordReader method convert.

@SuppressWarnings("unchecked")
protected Object convert(final Object value, final DataType dataType) {
    if (value == null) {
        return null;
    }
    if (value instanceof List) {
        final List<?> list = (List<?>) value;
        final Object[] array = new Object[list.size()];
        final DataType elementDataType;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.ARRAY) {
            elementDataType = ((ArrayDataType) dataType).getElementType();
        } else {
            elementDataType = null;
        }
        int i = 0;
        for (final Object val : list) {
            array[i++] = convert(val, elementDataType);
        }
        return array;
    }
    if (value instanceof Map) {
        final Map<String, ?> map = (Map<String, ?>) value;
        boolean record = false;
        for (final Object obj : map.values()) {
            if (obj instanceof JsonNode) {
                record = true;
            }
        }
        if (!record) {
            return value;
        }
        RecordSchema childSchema = null;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.RECORD) {
            childSchema = ((RecordDataType) dataType).getChildSchema();
        }
        if (childSchema == null) {
            childSchema = new SimpleRecordSchema(Collections.emptyList());
        }
        final Map<String, Object> values = new HashMap<>();
        for (final Map.Entry<String, ?> entry : map.entrySet()) {
            final String key = entry.getKey();
            final Object childValue = entry.getValue();
            final RecordField recordField = childSchema.getField(key).orElse(null);
            final DataType childDataType = recordField == null ? null : recordField.getDataType();
            values.put(key, convert(childValue, childDataType));
        }
        return new MapRecord(childSchema, values);
    }
    return value;
}

Example 10 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestCSVRecordReader method testExtraFieldNotInHeader.

@Test
public void testExtraFieldNotInHeader() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode, country";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, USA, North America";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    // test nextRecord does not contain a 'continent' field
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final CSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord(false, false);
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertEquals("USA", record.getValue("country"));
        assertEquals("North America", record.getValue("unknown_field_index_8"));
        assertNull(reader.nextRecord(false, false));
    }
}