Examples with FieldValue org.apache.nifi.record.path.FieldValue used on opensource projects

Example 16 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class LookupRecord method route.

@Override
protected Set<Relationship> route(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context, final Tuple<Map<String, RecordPath>, RecordPath> flowFileContext) {
    final Map<String, RecordPath> recordPaths = flowFileContext.getKey();
    final Map<String, Object> lookupCoordinates = new HashMap<>(recordPaths.size());
    for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
        final String coordinateKey = entry.getKey();
        final RecordPath recordPath = entry.getValue();
        final RecordPathResult pathResult = recordPath.evaluate(record);
        final List<FieldValue> lookupFieldValues = pathResult.getSelectedFields().filter(fieldVal -> fieldVal.getValue() != null).collect(Collectors.toList());
        if (lookupFieldValues.isEmpty()) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' did not match any fields in a record for {}; routing record to {}", new Object[] { coordinateKey, flowFile, rels });
            return rels;
        }
        if (lookupFieldValues.size() > 1) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' matched {} fields in a record for {}; routing record to {}", new Object[] { coordinateKey, lookupFieldValues.size(), flowFile, rels });
            return rels;
        }
        final FieldValue fieldValue = lookupFieldValues.get(0);
        final Object coordinateValue = (fieldValue.getValue() instanceof Number || fieldValue.getValue() instanceof Boolean) ? fieldValue.getValue() : DataTypeUtils.toString(fieldValue.getValue(), (String) null);
        lookupCoordinates.put(coordinateKey, coordinateValue);
    }
    final Optional<?> lookupValueOption;
    try {
        lookupValueOption = lookupService.lookup(lookupCoordinates);
    } catch (final Exception e) {
        throw new ProcessException("Failed to lookup coordinates " + lookupCoordinates + " in Lookup Service", e);
    }
    if (!lookupValueOption.isPresent()) {
        final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
        return rels;
    }
    // Ensure that the Record has the appropriate schema to account for the newly added values
    final RecordPath resultPath = flowFileContext.getValue();
    if (resultPath != null) {
        record.incorporateSchema(writeSchema);
        final Object lookupValue = lookupValueOption.get();
        final RecordPathResult resultPathResult = flowFileContext.getValue().evaluate(record);
        final String resultContentsValue = context.getProperty(RESULT_CONTENTS).getValue();
        if (RESULT_RECORD_FIELDS.getValue().equals(resultContentsValue) && lookupValue instanceof Record) {
            final Record lookupRecord = (Record) lookupValue;
            // Use wants to add all fields of the resultant Record to the specified Record Path.
            // If the destination Record Path returns to us a Record, then we will add all field values of
            // the Lookup Record to the destination Record. However, if the destination Record Path returns
            // something other than a Record, then we can't add the fields to it. We can only replace it,
            // because it doesn't make sense to add fields to anything but a Record.
            resultPathResult.getSelectedFields().forEach(fieldVal -> {
                final Object destinationValue = fieldVal.getValue();
                if (destinationValue instanceof Record) {
                    final Record destinationRecord = (Record) destinationValue;
                    for (final String fieldName : lookupRecord.getRawFieldNames()) {
                        final Object value = lookupRecord.getValue(fieldName);
                        destinationRecord.setValue(fieldName, value);
                    }
                } else {
                    final Optional<Record> parentOption = fieldVal.getParentRecord();
                    if (parentOption.isPresent()) {
                        parentOption.get().setValue(fieldVal.getField().getFieldName(), lookupRecord);
                    }
                }
            });
        } else {
            resultPathResult.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(lookupValue));
        }
    }
    final Set<Relationship> rels = routeToMatchedUnmatched ? MATCHED_COLLECTION : SUCCESS_COLLECTION;
    return rels;
}

Example 17 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class PartitionRecord method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
    final Map<String, RecordPath> recordPaths;
    try {
        recordPaths = context.getProperties().keySet().stream().filter(prop -> prop.isDynamic()).collect(Collectors.toMap(prop -> prop.getName(), prop -> getRecordPath(context, prop, flowFile)));
    } catch (final Exception e) {
        getLogger().error("Failed to compile RecordPath for {}; routing to failure", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final Map<RecordValueMap, RecordSetWriter> writerMap = new HashMap<>();
    try (final InputStream in = session.read(flowFile)) {
        final Map<String, String> originalAttributes = flowFile.getAttributes();
        final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger());
        final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema());
        Record record;
        while ((record = reader.nextRecord()) != null) {
            final Map<String, List<ValueWrapper>> recordMap = new HashMap<>();
            // Evaluate all of the RecordPath's for this Record
            for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
                final String propName = entry.getKey();
                final RecordPath recordPath = entry.getValue();
                final Stream<FieldValue> fieldValueStream = recordPath.evaluate(record).getSelectedFields();
                final List<ValueWrapper> fieldValues = fieldValueStream.map(fieldVal -> new ValueWrapper(fieldVal.getValue())).collect(Collectors.toList());
                recordMap.put(propName, fieldValues);
            }
            final RecordValueMap recordValueMap = new RecordValueMap(recordMap);
            // Get the RecordSetWriter that contains the same values for all RecordPaths - or create one if none exists.
            RecordSetWriter writer = writerMap.get(recordValueMap);
            if (writer == null) {
                final FlowFile childFlowFile = session.create(flowFile);
                recordValueMap.setFlowFile(childFlowFile);
                final OutputStream out = session.write(childFlowFile);
                writer = writerFactory.createWriter(getLogger(), writeSchema, out);
                writer.beginRecordSet();
                writerMap.put(recordValueMap, writer);
            }
            writer.write(record);
        }
        // For each RecordSetWriter, finish the record set and close the writer.
        for (final Map.Entry<RecordValueMap, RecordSetWriter> entry : writerMap.entrySet()) {
            final RecordValueMap valueMap = entry.getKey();
            final RecordSetWriter writer = entry.getValue();
            final WriteResult writeResult = writer.finishRecordSet();
            writer.close();
            final Map<String, String> attributes = new HashMap<>();
            attributes.putAll(valueMap.getAttributes());
            attributes.putAll(writeResult.getAttributes());
            attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
            attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType());
            FlowFile childFlowFile = valueMap.getFlowFile();
            childFlowFile = session.putAllAttributes(childFlowFile, attributes);
            session.adjustCounter("Record Processed", writeResult.getRecordCount(), false);
        }
    } catch (final Exception e) {
        for (final Map.Entry<RecordValueMap, RecordSetWriter> entry : writerMap.entrySet()) {
            final RecordValueMap valueMap = entry.getKey();
            final RecordSetWriter writer = entry.getValue();
            try {
                writer.close();
            } catch (final IOException e1) {
                getLogger().warn("Failed to close Record Writer for {}; some resources may not be cleaned up appropriately", new Object[] { flowFile, e1 });
            }
            session.remove(valueMap.getFlowFile());
        }
        getLogger().error("Failed to partition {}", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    // because we want to ensure that we are able to remove the child flowfiles in case of a failure.
    for (final RecordValueMap valueMap : writerMap.keySet()) {
        session.transfer(valueMap.getFlowFile(), REL_SUCCESS);
    }
    session.transfer(flowFile, REL_ORIGINAL);
}

Example 18 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class BinaryOperatorFilter method filter.

@Override
public Stream<FieldValue> filter(final RecordPathEvaluationContext context, final boolean invert) {
    final Stream<FieldValue> rhsStream = rhs.evaluate(context);
    final Optional<FieldValue> firstMatch = rhsStream.filter(fieldVal -> fieldVal.getValue() != null).findFirst();
    if (!firstMatch.isPresent()) {
        return Stream.empty();
    }
    final FieldValue fieldValue = firstMatch.get();
    final Object value = fieldValue.getValue();
    final Stream<FieldValue> lhsStream = lhs.evaluate(context);
    return lhsStream.filter(fieldVal -> {
        final boolean result = test(fieldVal, value);
        return invert ? !result : result;
    });
}

Example 19 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class MatchesRegex method test.

@Override
protected boolean test(final FieldValue fieldValue, final RecordPathEvaluationContext context) {
    final Pattern pattern;
    if (compiledPattern == null) {
        final Optional<FieldValue> fieldValueOption = regexPath.evaluate(context).findFirst();
        if (!fieldValueOption.isPresent()) {
            return false;
        }
        final Object value = fieldValueOption.get().getValue();
        if (value == null) {
            return false;
        }
        final String regex = DataTypeUtils.toString(value, (String) null);
        pattern = Pattern.compile(regex);
    } else {
        pattern = compiledPattern;
    }
    final String searchString = DataTypeUtils.toString(fieldValue.getValue(), (String) null);
    if (searchString == null) {
        return false;
    }
    return pattern.matcher(searchString).matches();
}

Example 20 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class ReplaceNull method evaluate.

@Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final Stream<FieldValue> fieldValues = recordPath.evaluate(context);
    return fieldValues.map(fv -> {
        if (fv.getValue() != null) {
            return fv;
        }
        final Optional<FieldValue> replacementOption = replacementValuePath.evaluate(context).findFirst();
        if (!replacementOption.isPresent()) {
            return fv;
        }
        final FieldValue replacementFieldValue = replacementOption.get();
        final Object replacementValue = replacementFieldValue.getValue();
        if (replacementValue == null) {
            return fv;
        }
        return new StandardFieldValue(replacementValue, fv.getField(), fv.getParent().orElse(null));
    });
}