Examples with RecordPath org.apache.nifi.record.path.RecordPath used on opensource projects

Example 1 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class LookupRecord method customValidate.

@Override
@SuppressWarnings("unchecked")
protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
    final Set<String> dynamicPropNames = validationContext.getProperties().keySet().stream().filter(prop -> prop.isDynamic()).map(prop -> prop.getName()).collect(Collectors.toSet());
    if (dynamicPropNames.isEmpty()) {
        return Collections.singleton(new ValidationResult.Builder().subject("User-Defined Properties").valid(false).explanation("At least one user-defined property must be specified.").build());
    }
    final Set<String> requiredKeys = validationContext.getProperty(LOOKUP_SERVICE).asControllerService(LookupService.class).getRequiredKeys();
    final Set<String> missingKeys = requiredKeys.stream().filter(key -> !dynamicPropNames.contains(key)).collect(Collectors.toSet());
    if (!missingKeys.isEmpty()) {
        final List<ValidationResult> validationResults = new ArrayList<>();
        for (final String missingKey : missingKeys) {
            final ValidationResult result = new ValidationResult.Builder().subject(missingKey).valid(false).explanation("The configured Lookup Services requires that a key be provided with the name '" + missingKey + "'. Please add a new property to this Processor with a name '" + missingKey + "' and provide a RecordPath that can be used to retrieve the appropriate value.").build();
            validationResults.add(result);
        }
        return validationResults;
    }
    return Collections.emptyList();
}

Example 2 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class LookupRecord method getFlowFileContext.

@Override
protected Tuple<Map<String, RecordPath>, RecordPath> getFlowFileContext(final FlowFile flowFile, final ProcessContext context) {
    final Map<String, RecordPath> recordPaths = new HashMap<>();
    for (final PropertyDescriptor prop : context.getProperties().keySet()) {
        if (!prop.isDynamic()) {
            continue;
        }
        final String pathText = context.getProperty(prop).evaluateAttributeExpressions(flowFile).getValue();
        final RecordPath lookupRecordPath = recordPathCache.getCompiled(pathText);
        recordPaths.put(prop.getName(), lookupRecordPath);
    }
    final RecordPath resultRecordPath;
    if (context.getProperty(RESULT_RECORD_PATH).isSet()) {
        final String resultPathText = context.getProperty(RESULT_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
        resultRecordPath = recordPathCache.getCompiled(resultPathText);
    } else {
        resultRecordPath = null;
    }
    return new Tuple<>(recordPaths, resultRecordPath);
}

Example 3 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class UpdateRecord method process.

@Override
protected Record process(Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context) {
    final boolean evaluateValueAsRecordPath = context.getProperty(REPLACEMENT_VALUE_STRATEGY).getValue().equals(RECORD_PATH_VALUES.getValue());
    // Incorporate the RecordSchema that we will use for writing records into the Schema that we have
    // for the record, because it's possible that the updates to the record will not be valid otherwise.
    record.incorporateSchema(writeSchema);
    for (final String recordPathText : recordPaths) {
        final RecordPath recordPath = recordPathCache.getCompiled(recordPathText);
        final RecordPathResult result = recordPath.evaluate(record);
        if (evaluateValueAsRecordPath) {
            final String replacementValue = context.getProperty(recordPathText).evaluateAttributeExpressions(flowFile).getValue();
            final RecordPath replacementRecordPath = recordPathCache.getCompiled(replacementValue);
            // If the RecordPath is a Relative Path, then we have to evaluate it against each FieldValue.
            if (replacementRecordPath.isAbsolute()) {
                record = processAbsolutePath(replacementRecordPath, result.getSelectedFields(), record);
            } else {
                record = processRelativePath(replacementRecordPath, result.getSelectedFields(), record);
            }
        } else {
            final PropertyValue replacementValue = context.getProperty(recordPathText);
            if (replacementValue.isExpressionLanguagePresent()) {
                final Map<String, String> fieldVariables = new HashMap<>();
                result.getSelectedFields().forEach(fieldVal -> {
                    fieldVariables.clear();
                    fieldVariables.put(FIELD_NAME, fieldVal.getField().getFieldName());
                    fieldVariables.put(FIELD_VALUE, DataTypeUtils.toString(fieldVal.getValue(), (String) null));
                    fieldVariables.put(FIELD_TYPE, fieldVal.getField().getDataType().getFieldType().name());
                    final String evaluatedReplacementVal = replacementValue.evaluateAttributeExpressions(flowFile, fieldVariables).getValue();
                    fieldVal.updateValue(evaluatedReplacementVal);
                });
            } else {
                final String evaluatedReplacementVal = replacementValue.getValue();
                result.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(evaluatedReplacementVal));
            }
        }
    }
    return record;
}

Example 4 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class RecordPathCache method getCompiled.

public RecordPath getCompiled(final String path) {
    RecordPath compiled;
    synchronized (this) {
        compiled = compiledRecordPaths.get(path);
    }
    if (compiled != null) {
        return compiled;
    }
    compiled = RecordPath.compile(path);
    synchronized (this) {
        final RecordPath existing = compiledRecordPaths.putIfAbsent(path, compiled);
        if (existing != null) {
            compiled = existing;
        }
    }
    return compiled;
}

Example 5 with RecordPath

use of org.apache.nifi.record.path.RecordPath in project nifi by apache.

the class PutElasticsearchHttpRecord method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions(flowFile).getValue();
    OkHttpClient okHttpClient = getClient();
    final ComponentLog logger = getLogger();
    final String baseUrl = trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
    HttpUrl.Builder urlBuilder = HttpUrl.parse(baseUrl).newBuilder().addPathSegment("_bulk");
    // Find the user-added properties and set them as query parameters on the URL
    for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
        PropertyDescriptor pd = property.getKey();
        if (pd.isDynamic()) {
            if (property.getValue() != null) {
                urlBuilder = urlBuilder.addQueryParameter(pd.getName(), context.getProperty(pd).evaluateAttributeExpressions().getValue());
            }
        }
    }
    final URL url = urlBuilder.build().url();
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(index)) {
        logger.error("No value for index in for {}, transferring to failure", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(indexOp)) {
        logger.error("No Index operation specified for {}, transferring to failure.", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    switch(indexOp.toLowerCase()) {
        case "index":
        case "update":
        case "upsert":
        case "delete":
            break;
        default:
            logger.error("Index operation {} not supported for {}, transferring to failure.", new Object[] { indexOp, flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
    }
    this.nullSuppression = context.getProperty(SUPPRESS_NULLS).getValue();
    final String id_path = context.getProperty(ID_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final RecordPath recordPath = StringUtils.isEmpty(id_path) ? null : recordPathCache.getCompiled(id_path);
    final StringBuilder sb = new StringBuilder();
    try (final InputStream in = session.read(flowFile);
        final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) {
        Record record;
        while ((record = reader.nextRecord()) != null) {
            final String id;
            if (recordPath != null) {
                Optional<FieldValue> idPathValue = recordPath.evaluate(record).getSelectedFields().findFirst();
                if (!idPathValue.isPresent() || idPathValue.get().getValue() == null) {
                    throw new IdentifierNotFoundException("Identifier Record Path specified but no value was found, transferring {} to failure.");
                }
                id = idPathValue.get().getValue().toString();
            } else {
                id = null;
            }
            // a missing ID indicates one is to be auto-generated by Elasticsearch
            if (id == null && !indexOp.equalsIgnoreCase("index")) {
                throw new IdentifierNotFoundException("Index operation {} requires a valid identifier value from a flow file attribute, transferring to failure.");
            }
            final StringBuilder json = new StringBuilder();
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            JsonGenerator generator = factory.createJsonGenerator(out);
            writeRecord(record, record.getSchema(), generator);
            generator.flush();
            generator.close();
            json.append(out.toString());
            if (indexOp.equalsIgnoreCase("index")) {
                sb.append("{\"index\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\"");
                if (!StringUtils.isEmpty(id)) {
                    sb.append(", \"_id\": \"");
                    sb.append(id);
                    sb.append("\"");
                }
                sb.append("}}\n");
                sb.append(json);
                sb.append("\n");
            } else if (indexOp.equalsIgnoreCase("upsert") || indexOp.equalsIgnoreCase("update")) {
                sb.append("{\"update\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
                sb.append("{\"doc\": ");
                sb.append(json);
                sb.append(", \"doc_as_upsert\": ");
                sb.append(indexOp.equalsIgnoreCase("upsert"));
                sb.append(" }\n");
            } else if (indexOp.equalsIgnoreCase("delete")) {
                sb.append("{\"delete\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
            }
        }
    } catch (IdentifierNotFoundException infe) {
        logger.error(infe.getMessage(), new Object[] { flowFile });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    } catch (final IOException | SchemaNotFoundException | MalformedRecordException e) {
        logger.error("Could not parse incoming data", e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), sb.toString());
    final Response getResponse;
    try {
        getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "PUT", requestBody);
    } catch (final Exception e) {
        logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final int statusCode = getResponse.code();
    if (isSuccess(statusCode)) {
        ResponseBody responseBody = getResponse.body();
        try {
            final byte[] bodyBytes = responseBody.bytes();
            JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
            boolean errors = responseJson.get("errors").asBoolean(false);
            // ES has no rollback, so if errors occur, log them and route the whole flow file to failure
            if (errors) {
                ArrayNode itemNodeArray = (ArrayNode) responseJson.get("items");
                if (itemNodeArray.size() > 0) {
                    // All items are returned whether they succeeded or failed, so iterate through the item array
                    // at the same time as the flow file list, moving each to success or failure accordingly,
                    // but only keep the first error for logging
                    String errorReason = null;
                    for (int i = itemNodeArray.size() - 1; i >= 0; i--) {
                        JsonNode itemNode = itemNodeArray.get(i);
                        int status = itemNode.findPath("status").asInt();
                        if (!isSuccess(status)) {
                            if (errorReason == null) {
                                // Use "result" if it is present; this happens for status codes like 404 Not Found, which may not have an error/reason
                                String reason = itemNode.findPath("//result").asText();
                                if (StringUtils.isEmpty(reason)) {
                                    // If there was no result, we expect an error with a string description in the "reason" field
                                    reason = itemNode.findPath("//error/reason").asText();
                                }
                                errorReason = reason;
                                logger.error("Failed to process {} due to {}, transferring to failure", new Object[] { flowFile, errorReason });
                            }
                        }
                    }
                }
                session.transfer(flowFile, REL_FAILURE);
            } else {
                session.transfer(flowFile, REL_SUCCESS);
                session.getProvenanceReporter().send(flowFile, url.toString());
            }
        } catch (IOException ioe) {
            // Something went wrong when parsing the response, log the error and route to failure
            logger.error("Error parsing Bulk API response: {}", new Object[] { ioe.getMessage() }, ioe);
            session.transfer(flowFile, REL_FAILURE);
            context.yield();
        }
    } else if (statusCode / 100 == 5) {
        // 5xx -> RETRY, but a server error might last a while, so yield
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_RETRY);
        context.yield();
    } else {
        // 1xx, 3xx, 4xx, etc. -> NO RETRY
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_FAILURE);
    }
    getResponse.close();
}