Examples with RecordPath org.apache.nifi.record.path.RecordPath used on opensource projects

Example 6 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class LookupRecord method route.

@Override
protected Set<Relationship> route(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context, final Tuple<Map<String, RecordPath>, RecordPath> flowFileContext) {
    final Map<String, RecordPath> recordPaths = flowFileContext.getKey();
    final Map<String, Object> lookupCoordinates = new HashMap<>(recordPaths.size());
    for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
        final String coordinateKey = entry.getKey();
        final RecordPath recordPath = entry.getValue();
        final RecordPathResult pathResult = recordPath.evaluate(record);
        final List<FieldValue> lookupFieldValues = pathResult.getSelectedFields().filter(fieldVal -> fieldVal.getValue() != null).collect(Collectors.toList());
        if (lookupFieldValues.isEmpty()) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' did not match any fields in a record for {}; routing record to {}", new Object[] { coordinateKey, flowFile, rels });
            return rels;
        }
        if (lookupFieldValues.size() > 1) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' matched {} fields in a record for {}; routing record to {}", new Object[] { coordinateKey, lookupFieldValues.size(), flowFile, rels });
            return rels;
        }
        final FieldValue fieldValue = lookupFieldValues.get(0);
        final Object coordinateValue = (fieldValue.getValue() instanceof Number || fieldValue.getValue() instanceof Boolean) ? fieldValue.getValue() : DataTypeUtils.toString(fieldValue.getValue(), (String) null);
        lookupCoordinates.put(coordinateKey, coordinateValue);
    }
    final Optional<?> lookupValueOption;
    try {
        lookupValueOption = lookupService.lookup(lookupCoordinates);
    } catch (final Exception e) {
        throw new ProcessException("Failed to lookup coordinates " + lookupCoordinates + " in Lookup Service", e);
    }
    if (!lookupValueOption.isPresent()) {
        final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
        return rels;
    }
    // Ensure that the Record has the appropriate schema to account for the newly added values
    final RecordPath resultPath = flowFileContext.getValue();
    if (resultPath != null) {
        record.incorporateSchema(writeSchema);
        final Object lookupValue = lookupValueOption.get();
        final RecordPathResult resultPathResult = flowFileContext.getValue().evaluate(record);
        final String resultContentsValue = context.getProperty(RESULT_CONTENTS).getValue();
        if (RESULT_RECORD_FIELDS.getValue().equals(resultContentsValue) && lookupValue instanceof Record) {
            final Record lookupRecord = (Record) lookupValue;
            // Use wants to add all fields of the resultant Record to the specified Record Path.
            // If the destination Record Path returns to us a Record, then we will add all field values of
            // the Lookup Record to the destination Record. However, if the destination Record Path returns
            // something other than a Record, then we can't add the fields to it. We can only replace it,
            // because it doesn't make sense to add fields to anything but a Record.
            resultPathResult.getSelectedFields().forEach(fieldVal -> {
                final Object destinationValue = fieldVal.getValue();
                if (destinationValue instanceof Record) {
                    final Record destinationRecord = (Record) destinationValue;
                    for (final String fieldName : lookupRecord.getRawFieldNames()) {
                        final Object value = lookupRecord.getValue(fieldName);
                        destinationRecord.setValue(fieldName, value);
                    }
                } else {
                    final Optional<Record> parentOption = fieldVal.getParentRecord();
                    if (parentOption.isPresent()) {
                        parentOption.get().setValue(fieldVal.getField().getFieldName(), lookupRecord);
                    }
                }
            });
        } else {
            resultPathResult.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(lookupValue));
        }
    }
    final Set<Relationship> rels = routeToMatchedUnmatched ? MATCHED_COLLECTION : SUCCESS_COLLECTION;
    return rels;
}

Example 7 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class PartitionRecord method getRecordPath.

private RecordPath getRecordPath(final ProcessContext context, final PropertyDescriptor prop, final FlowFile flowFile) {
    final String pathText = context.getProperty(prop).evaluateAttributeExpressions(flowFile).getValue();
    final RecordPath recordPath = recordPathCache.getCompiled(pathText);
    return recordPath;
}

Example 8 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class PartitionRecord method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
    final Map<String, RecordPath> recordPaths;
    try {
        recordPaths = context.getProperties().keySet().stream().filter(prop -> prop.isDynamic()).collect(Collectors.toMap(prop -> prop.getName(), prop -> getRecordPath(context, prop, flowFile)));
    } catch (final Exception e) {
        getLogger().error("Failed to compile RecordPath for {}; routing to failure", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final Map<RecordValueMap, RecordSetWriter> writerMap = new HashMap<>();
    try (final InputStream in = session.read(flowFile)) {
        final Map<String, String> originalAttributes = flowFile.getAttributes();
        final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger());
        final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema());
        Record record;
        while ((record = reader.nextRecord()) != null) {
            final Map<String, List<ValueWrapper>> recordMap = new HashMap<>();
            // Evaluate all of the RecordPath's for this Record
            for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
                final String propName = entry.getKey();
                final RecordPath recordPath = entry.getValue();
                final Stream<FieldValue> fieldValueStream = recordPath.evaluate(record).getSelectedFields();
                final List<ValueWrapper> fieldValues = fieldValueStream.map(fieldVal -> new ValueWrapper(fieldVal.getValue())).collect(Collectors.toList());
                recordMap.put(propName, fieldValues);
            }
            final RecordValueMap recordValueMap = new RecordValueMap(recordMap);
            // Get the RecordSetWriter that contains the same values for all RecordPaths - or create one if none exists.
            RecordSetWriter writer = writerMap.get(recordValueMap);
            if (writer == null) {
                final FlowFile childFlowFile = session.create(flowFile);
                recordValueMap.setFlowFile(childFlowFile);
                final OutputStream out = session.write(childFlowFile);
                writer = writerFactory.createWriter(getLogger(), writeSchema, out);
                writer.beginRecordSet();
                writerMap.put(recordValueMap, writer);
            }
            writer.write(record);
        }
        // For each RecordSetWriter, finish the record set and close the writer.
        for (final Map.Entry<RecordValueMap, RecordSetWriter> entry : writerMap.entrySet()) {
            final RecordValueMap valueMap = entry.getKey();
            final RecordSetWriter writer = entry.getValue();
            final WriteResult writeResult = writer.finishRecordSet();
            writer.close();
            final Map<String, String> attributes = new HashMap<>();
            attributes.putAll(valueMap.getAttributes());
            attributes.putAll(writeResult.getAttributes());
            attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
            attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType());
            FlowFile childFlowFile = valueMap.getFlowFile();
            childFlowFile = session.putAllAttributes(childFlowFile, attributes);
            session.adjustCounter("Record Processed", writeResult.getRecordCount(), false);
        }
    } catch (final Exception e) {
        for (final Map.Entry<RecordValueMap, RecordSetWriter> entry : writerMap.entrySet()) {
            final RecordValueMap valueMap = entry.getKey();
            final RecordSetWriter writer = entry.getValue();
            try {
                writer.close();
            } catch (final IOException e1) {
                getLogger().warn("Failed to close Record Writer for {}; some resources may not be cleaned up appropriately", new Object[] { flowFile, e1 });
            }
            session.remove(valueMap.getFlowFile());
        }
        getLogger().error("Failed to partition {}", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    // because we want to ensure that we are able to remove the child flowfiles in case of a failure.
    for (final RecordValueMap valueMap : writerMap.keySet()) {
        session.transfer(valueMap.getFlowFile(), REL_SUCCESS);
    }
    session.transfer(flowFile, REL_ORIGINAL);
}