Examples with FieldValue org.apache.nifi.record.path.FieldValue used on opensource projects

Example 11 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class CurrentFieldPath method evaluate.

@Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final FieldValue contextNode = context.getContextNode();
    if (contextNode != null) {
        return Stream.of(contextNode);
    }
    final RecordPathSegment parentPath = getParentPath();
    if (parentPath == null) {
        return Stream.of(context.getContextNode());
    } else {
        return parentPath.evaluate(context);
    }
}

Example 12 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class DescendantFieldPath method findDescendants.

private List<FieldValue> findDescendants(final FieldValue fieldValue) {
    if (fieldValue == null || fieldValue.getValue() == null) {
        return Collections.emptyList();
    }
    if (!Filters.isRecord(fieldValue)) {
        return Collections.emptyList();
    }
    final Record record = (Record) fieldValue.getValue();
    final List<FieldValue> matchingValues = new ArrayList<>();
    for (final RecordField childField : record.getSchema().getFields()) {
        if (childField.getFieldName().equals(descendantName) || childField.getAliases().contains(descendantName)) {
            final Object value = record.getValue(descendantName);
            if (value != null) {
                final FieldValue descendantFieldValue = new StandardFieldValue(value, childField, fieldValue);
                matchingValues.add(descendantFieldValue);
            }
        }
        final Object recordValue = record.getValue(childField.getFieldName());
        if (recordValue == null) {
            continue;
        }
        if (Filters.isRecord(childField.getDataType(), recordValue)) {
            final FieldValue childFieldValue = new StandardFieldValue(recordValue, childField, fieldValue);
            matchingValues.addAll(findDescendants(childFieldValue));
        }
    }
    return matchingValues;
}

Example 13 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class RecordPathSegment method evaluate.

@Override
public final RecordPathResult evaluate(final Record record) {
    final RecordPathEvaluationContext context = new StandardRecordPathEvaluationContext(record);
    final Stream<FieldValue> selectedFields = evaluate(context);
    return new RecordPathResult() {

        @Override
        public String getPath() {
            return RecordPathSegment.this.getPath();
        }

        @Override
        public Stream<FieldValue> getSelectedFields() {
            return selectedFields;
        }
    };
}

Example 14 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class SingularMapKeyPath method evaluate.

@Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
    return parentResult.filter(Filters.fieldTypeFilter(RecordFieldType.MAP)).map(fieldValue -> {
        final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
        final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType);
        return new MapEntryFieldValue(getMapValue(fieldValue), elementField, fieldValue, mapKey);
    });
}

Example 15 with FieldValue

use of org.apache.nifi.record.path.FieldValue in project nifi by apache.

the class PutElasticsearchHttpRecord method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions(flowFile).getValue();
    OkHttpClient okHttpClient = getClient();
    final ComponentLog logger = getLogger();
    final String baseUrl = trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
    HttpUrl.Builder urlBuilder = HttpUrl.parse(baseUrl).newBuilder().addPathSegment("_bulk");
    // Find the user-added properties and set them as query parameters on the URL
    for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
        PropertyDescriptor pd = property.getKey();
        if (pd.isDynamic()) {
            if (property.getValue() != null) {
                urlBuilder = urlBuilder.addQueryParameter(pd.getName(), context.getProperty(pd).evaluateAttributeExpressions().getValue());
            }
        }
    }
    final URL url = urlBuilder.build().url();
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(index)) {
        logger.error("No value for index in for {}, transferring to failure", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(indexOp)) {
        logger.error("No Index operation specified for {}, transferring to failure.", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    switch(indexOp.toLowerCase()) {
        case "index":
        case "update":
        case "upsert":
        case "delete":
            break;
        default:
            logger.error("Index operation {} not supported for {}, transferring to failure.", new Object[] { indexOp, flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
    }
    this.nullSuppression = context.getProperty(SUPPRESS_NULLS).getValue();
    final String id_path = context.getProperty(ID_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final RecordPath recordPath = StringUtils.isEmpty(id_path) ? null : recordPathCache.getCompiled(id_path);
    final StringBuilder sb = new StringBuilder();
    try (final InputStream in = session.read(flowFile);
        final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) {
        Record record;
        while ((record = reader.nextRecord()) != null) {
            final String id;
            if (recordPath != null) {
                Optional<FieldValue> idPathValue = recordPath.evaluate(record).getSelectedFields().findFirst();
                if (!idPathValue.isPresent() || idPathValue.get().getValue() == null) {
                    throw new IdentifierNotFoundException("Identifier Record Path specified but no value was found, transferring {} to failure.");
                }
                id = idPathValue.get().getValue().toString();
            } else {
                id = null;
            }
            // a missing ID indicates one is to be auto-generated by Elasticsearch
            if (id == null && !indexOp.equalsIgnoreCase("index")) {
                throw new IdentifierNotFoundException("Index operation {} requires a valid identifier value from a flow file attribute, transferring to failure.");
            }
            final StringBuilder json = new StringBuilder();
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            JsonGenerator generator = factory.createJsonGenerator(out);
            writeRecord(record, record.getSchema(), generator);
            generator.flush();
            generator.close();
            json.append(out.toString());
            if (indexOp.equalsIgnoreCase("index")) {
                sb.append("{\"index\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\"");
                if (!StringUtils.isEmpty(id)) {
                    sb.append(", \"_id\": \"");
                    sb.append(id);
                    sb.append("\"");
                }
                sb.append("}}\n");
                sb.append(json);
                sb.append("\n");
            } else if (indexOp.equalsIgnoreCase("upsert") || indexOp.equalsIgnoreCase("update")) {
                sb.append("{\"update\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
                sb.append("{\"doc\": ");
                sb.append(json);
                sb.append(", \"doc_as_upsert\": ");
                sb.append(indexOp.equalsIgnoreCase("upsert"));
                sb.append(" }\n");
            } else if (indexOp.equalsIgnoreCase("delete")) {
                sb.append("{\"delete\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
            }
        }
    } catch (IdentifierNotFoundException infe) {
        logger.error(infe.getMessage(), new Object[] { flowFile });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    } catch (final IOException | SchemaNotFoundException | MalformedRecordException e) {
        logger.error("Could not parse incoming data", e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), sb.toString());
    final Response getResponse;
    try {
        getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "PUT", requestBody);
    } catch (final Exception e) {
        logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final int statusCode = getResponse.code();
    if (isSuccess(statusCode)) {
        ResponseBody responseBody = getResponse.body();
        try {
            final byte[] bodyBytes = responseBody.bytes();
            JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
            boolean errors = responseJson.get("errors").asBoolean(false);
            // ES has no rollback, so if errors occur, log them and route the whole flow file to failure
            if (errors) {
                ArrayNode itemNodeArray = (ArrayNode) responseJson.get("items");
                if (itemNodeArray.size() > 0) {
                    // All items are returned whether they succeeded or failed, so iterate through the item array
                    // at the same time as the flow file list, moving each to success or failure accordingly,
                    // but only keep the first error for logging
                    String errorReason = null;
                    for (int i = itemNodeArray.size() - 1; i >= 0; i--) {
                        JsonNode itemNode = itemNodeArray.get(i);
                        int status = itemNode.findPath("status").asInt();
                        if (!isSuccess(status)) {
                            if (errorReason == null) {
                                // Use "result" if it is present; this happens for status codes like 404 Not Found, which may not have an error/reason
                                String reason = itemNode.findPath("//result").asText();
                                if (StringUtils.isEmpty(reason)) {
                                    // If there was no result, we expect an error with a string description in the "reason" field
                                    reason = itemNode.findPath("//error/reason").asText();
                                }
                                errorReason = reason;
                                logger.error("Failed to process {} due to {}, transferring to failure", new Object[] { flowFile, errorReason });
                            }
                        }
                    }
                }
                session.transfer(flowFile, REL_FAILURE);
            } else {
                session.transfer(flowFile, REL_SUCCESS);
                session.getProvenanceReporter().send(flowFile, url.toString());
            }
        } catch (IOException ioe) {
            // Something went wrong when parsing the response, log the error and route to failure
            logger.error("Error parsing Bulk API response: {}", new Object[] { ioe.getMessage() }, ioe);
            session.transfer(flowFile, REL_FAILURE);
            context.yield();
        }
    } else if (statusCode / 100 == 5) {
        // 5xx -> RETRY, but a server error might last a while, so yield
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_RETRY);
        context.yield();
    } else {
        // 1xx, 3xx, 4xx, etc. -> NO RETRY
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_FAILURE);
    }
    getResponse.close();
}