Examples with Rule org.apache.nifi.update.attributes.Rule used on opensource projects

Example 6 with Rule

use of org.apache.nifi.update.attributes.Rule in project nifi by apache.

the class UpdateAttribute method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final ComponentLog logger = getLogger();
    final Criteria criteria = criteriaCache.get();
    FlowFile incomingFlowFile = session.get();
    if (incomingFlowFile == null) {
        return;
    }
    // record which rule should be applied to which flow file - when operating
    // in 'use clone' mode, this collection will contain a number of entries
    // that map to single element lists. this is because the original flowfile
    // is cloned for each matching rule. in 'use original' mode, this collection
    // will contain a single entry that maps a list of multiple rules. this is
    // because is the original flowfile is used for all matching rules. in this
    // case the order of the matching rules is preserved in the list
    final Map<FlowFile, List<Rule>> matchedRules = new HashMap<>();
    final Map<String, String> stateInitialAttributes;
    final Map<String, String> stateWorkingAttributes;
    StateMap stateMap = null;
    try {
        if (stateful) {
            stateMap = context.getStateManager().getState(Scope.LOCAL);
            stateInitialAttributes = stateMap.toMap();
            stateWorkingAttributes = new HashMap<>(stateMap.toMap());
        } else {
            stateInitialAttributes = null;
            stateWorkingAttributes = null;
        }
    } catch (IOException e) {
        logger.error("Failed to get the initial state when processing {}; transferring FlowFile back to its incoming queue", new Object[] { incomingFlowFile }, e);
        session.transfer(incomingFlowFile);
        context.yield();
        return;
    }
    Map<String, Action> defaultActions = this.defaultActions;
    List<FlowFile> flowFilesToTransfer = new LinkedList<>();
    // if there is update criteria specified, evaluate it
    if (criteria != null && evaluateCriteria(session, context, criteria, incomingFlowFile, matchedRules, stateInitialAttributes)) {
        // apply the actions for each rule and transfer the flowfile
        for (final Map.Entry<FlowFile, List<Rule>> entry : matchedRules.entrySet()) {
            FlowFile match = entry.getKey();
            final List<Rule> rules = entry.getValue();
            boolean updateWorking = incomingFlowFile.equals(match);
            // execute each matching rule(s)
            match = executeActions(session, context, rules, defaultActions, match, stateInitialAttributes, stateWorkingAttributes);
            if (updateWorking) {
                incomingFlowFile = match;
            }
            if (debugEnabled) {
                logger.debug("Updated attributes for {}; transferring to '{}'", new Object[] { match, REL_SUCCESS.getName() });
            }
            // add the match to the list to transfer
            flowFilesToTransfer.add(match);
        }
    } else {
        // Either we're running without any rules or the FlowFile didn't match any
        incomingFlowFile = executeActions(session, context, null, defaultActions, incomingFlowFile, stateInitialAttributes, stateWorkingAttributes);
        if (debugEnabled) {
            logger.debug("Updated attributes for {}; transferring to '{}'", new Object[] { incomingFlowFile, REL_SUCCESS.getName() });
        }
        // add the flowfile to the list to transfer
        flowFilesToTransfer.add(incomingFlowFile);
    }
    if (stateInitialAttributes != null) {
        try {
            // Able to use "equals()" since we're just checking if the map was modified at all
            if (!stateWorkingAttributes.equals(stateInitialAttributes)) {
                boolean setState = context.getStateManager().replace(stateMap, stateWorkingAttributes, Scope.LOCAL);
                if (!setState) {
                    logger.warn("Failed to update the state after successfully processing {} due to having an old version of the StateMap. This is normally due to multiple threads running at " + "once; transferring to '{}'", new Object[] { incomingFlowFile, REL_FAILED_SET_STATE.getName() });
                    flowFilesToTransfer.remove(incomingFlowFile);
                    if (flowFilesToTransfer.size() > 0) {
                        session.remove(flowFilesToTransfer);
                    }
                    session.transfer(incomingFlowFile, REL_FAILED_SET_STATE);
                    return;
                }
            }
        } catch (IOException e) {
            logger.error("Failed to set the state after successfully processing {} due a failure when setting the state. This is normally due to multiple threads running at " + "once; transferring to '{}'", new Object[] { incomingFlowFile, REL_FAILED_SET_STATE.getName() }, e);
            flowFilesToTransfer.remove(incomingFlowFile);
            if (flowFilesToTransfer.size() > 0) {
                session.remove(flowFilesToTransfer);
            }
            session.transfer(incomingFlowFile, REL_FAILED_SET_STATE);
            context.yield();
            return;
        }
    }
    for (FlowFile toTransfer : flowFilesToTransfer) {
        session.getProvenanceReporter().modifyAttributes(toTransfer);
    }
    session.transfer(flowFilesToTransfer, REL_SUCCESS);
}

Example 7 with Rule

use of org.apache.nifi.update.attributes.Rule in project nifi by apache.

the class UpdateAttribute method customValidate.

@Override
protected Collection<ValidationResult> customValidate(final ValidationContext context) {
    final List<ValidationResult> reasons = new ArrayList<>(super.customValidate(context));
    if (!context.getProperty(STORE_STATE).getValue().equals(DO_NOT_STORE_STATE)) {
        String initValue = context.getProperty(STATEFUL_VARIABLES_INIT_VALUE).getValue();
        if (initValue == null) {
            reasons.add(new ValidationResult.Builder().subject(STATEFUL_VARIABLES_INIT_VALUE.getDisplayName()).valid(false).explanation("initial state value must be set if the processor is configured to store state.").build());
        }
    }
    Criteria criteria = null;
    try {
        criteria = CriteriaSerDe.deserialize(context.getAnnotationData());
    } catch (IllegalArgumentException iae) {
        reasons.add(new ValidationResult.Builder().valid(false).explanation("Unable to deserialize the update criteria." + iae.getMessage()).build());
    }
    // if there is criteria, validate it
    if (criteria != null) {
        final List<Rule> rules = criteria.getRules();
        if (rules == null) {
            reasons.add(new ValidationResult.Builder().valid(false).explanation("Update criteria has been specified by no rules were found.").build());
        } else {
            // validate the each rule
            for (final Rule rule : rules) {
                if (rule.getName() == null || rule.getName().trim().isEmpty()) {
                    reasons.add(new ValidationResult.Builder().valid(false).explanation("A rule name was not specified.").build());
                }
                // validate each condition
                final Set<Condition> conditions = rule.getConditions();
                if (conditions == null) {
                    reasons.add(new ValidationResult.Builder().valid(false).explanation(String.format("No conditions for rule '%s' found.", rule.getName())).build());
                } else {
                    for (final Condition condition : conditions) {
                        if (condition.getExpression() == null) {
                            reasons.add(new ValidationResult.Builder().valid(false).explanation(String.format("No expression for a condition in rule '%s' was found.", rule.getName())).build());
                        } else {
                            final String expression = condition.getExpression().trim();
                            reasons.add(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.BOOLEAN, false).validate(String.format("Condition for rule '%s'.", rule.getName()), expression, context));
                        }
                    }
                }
                // validate each action
                final Set<Action> actions = rule.getActions();
                if (actions == null) {
                    reasons.add(new ValidationResult.Builder().valid(false).explanation(String.format("No actions for rule '%s' found.", rule.getName())).build());
                } else {
                    for (final Action action : actions) {
                        if (action.getAttribute() == null) {
                            reasons.add(new ValidationResult.Builder().valid(false).explanation(String.format("An action in rule '%s' is missing the attribute name.", rule.getName())).build());
                        } else if (action.getValue() == null) {
                            reasons.add(new ValidationResult.Builder().valid(false).explanation(String.format("No value for attribute '%s' in rule '%s' was found.", action.getAttribute(), rule.getName())).build());
                        } else {
                            reasons.add(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING, true).validate(String.format("Action for rule '%s'.", rule.getName()), action.getValue(), context));
                        }
                    }
                }
            }
        }
    }
    return reasons;
}

Example 8 with Rule

use of org.apache.nifi.update.attributes.Rule in project nifi by apache.

the class UpdateAttribute method executeActions.

// Executes the specified action on the specified flowfile.
private FlowFile executeActions(final ProcessSession session, final ProcessContext context, final List<Rule> rules, final Map<String, Action> defaultActions, final FlowFile flowfile, final Map<String, String> stateInitialAttributes, final Map<String, String> stateWorkingAttributes) {
    final ComponentLog logger = getLogger();
    final Map<String, Action> actions = new HashMap<>(defaultActions);
    final String ruleName = (rules == null || rules.isEmpty()) ? "default" : rules.get(rules.size() - 1).getName();
    // if a rule matched, get its actions and possible overwrite the default ones
    if (rules != null && rules.size() > 0) {
        // subsequent matching rules will take precedence over previously matching rules and default values
        for (final Rule rule : rules) {
            for (final Action action : rule.getActions()) {
                // store the action and overwrite the previous value (from the defaults or a previously matching rule)
                actions.put(action.getAttribute(), action);
            }
        }
        // add an action for the matched rule - when matching multiple rules against
        // the original flowfile (use original) this will leave the last matching
        // rule's name as the value of this attribute. this decision was made since
        // this would be the behavior if they user chained multiple UpdateAttributes
        // together with 'use clone' specified
        final Action matchedRuleAction = new Action();
        matchedRuleAction.setAttribute(getClass().getSimpleName() + ".matchedRule");
        matchedRuleAction.setValue(ruleName);
        actions.put(matchedRuleAction.getAttribute(), matchedRuleAction);
    }
    // attribute values that will be applied to the flow file
    final Map<String, String> attributesToUpdate = new HashMap<>(actions.size());
    final Set<String> attributesToDelete = new HashSet<>(actions.size());
    // go through each action
    boolean debugEnabled = this.debugEnabled;
    for (final Action action : actions.values()) {
        String attribute = action.getAttribute();
        if (DELETE_ATTRIBUTES_EXPRESSION_NAME.equals(attribute)) {
            try {
                final String actionValue = action.getValue();
                final String regex = (actionValue == null) ? null : getPropertyValue(actionValue, context).evaluateAttributeExpressions(flowfile).getValue();
                if (regex != null) {
                    Pattern pattern = Pattern.compile(regex);
                    final Set<String> attributeKeys = flowfile.getAttributes().keySet();
                    for (final String key : attributeKeys) {
                        if (pattern.matcher(key).matches()) {
                            // log if appropriate
                            if (debugEnabled) {
                                logger.debug(String.format("%s deleting attribute '%s' for %s per regex '%s'.", this, key, flowfile, regex));
                            }
                            attributesToDelete.add(key);
                        }
                    }
                    // No point in updating if they will be removed
                    attributesToUpdate.keySet().removeAll(attributesToDelete);
                }
            } catch (final Exception e) {
                logger.error(String.format("Unable to delete attribute '%s' while processing FlowFile '%s' .", attribute, flowfile));
                throw new ProcessException(String.format("Unable to delete attribute '%s': %s.", attribute, e), e);
            }
        } else {
            boolean notDeleted = !attributesToDelete.contains(attribute);
            boolean setStatefulAttribute = stateInitialAttributes != null && !attribute.equals("UpdateAttribute.matchedRule");
            if (notDeleted || setStatefulAttribute) {
                try {
                    final String newAttributeValue = getPropertyValue(action.getValue(), context).evaluateAttributeExpressions(flowfile, null, null, stateInitialAttributes).getValue();
                    // log if appropriate
                    if (debugEnabled) {
                        logger.debug(String.format("%s setting attribute '%s' = '%s' for %s per rule '%s'.", this, attribute, newAttributeValue, flowfile, ruleName));
                    }
                    if (setStatefulAttribute) {
                        stateWorkingAttributes.put(attribute, newAttributeValue);
                    }
                    // No point in updating if it will be removed
                    if (notDeleted) {
                        attributesToUpdate.put(attribute, newAttributeValue);
                    }
                // Capture Exception thrown when evaluating the Expression Language
                } catch (final Exception e) {
                    logger.error(String.format("Could not evaluate the FlowFile '%s' against expression '%s' " + "defined by DynamicProperty '%s' due to '%s'", flowfile, action.getValue(), attribute, e.getLocalizedMessage()));
                    throw new ProcessException(String.format("Unable to evaluate new value for attribute '%s': %s.", attribute, e), e);
                }
            }
        }
    }
    // If the 'alternate.identifier' attribute is added, then we want to create an ADD_INFO provenance event.
    final String alternateIdentifierAdd = attributesToUpdate.get(CoreAttributes.ALTERNATE_IDENTIFIER.key());
    if (alternateIdentifierAdd != null) {
        try {
            final URI uri = new URI(alternateIdentifierAdd);
            final String namespace = uri.getScheme();
            if (namespace != null) {
                final String identifier = alternateIdentifierAdd.substring(Math.min(namespace.length() + 1, alternateIdentifierAdd.length() - 1));
                session.getProvenanceReporter().associate(flowfile, namespace, identifier);
            }
        } catch (final URISyntaxException e) {
        }
    }
    // update and delete the FlowFile attributes
    FlowFile returnFlowfile = flowfile;
    if (attributesToUpdate.size() > 0) {
        returnFlowfile = session.putAllAttributes(returnFlowfile, attributesToUpdate);
    }
    if (attributesToDelete.size() > 0) {
        returnFlowfile = session.removeAllAttributes(returnFlowfile, attributesToDelete);
    }
    return returnFlowfile;
}

Example 9 with Rule

use of org.apache.nifi.update.attributes.Rule in project nifi by apache.

the class UpdateAttribute method search.

@Override
public Collection<SearchResult> search(final SearchContext context) {
    final String term = context.getSearchTerm();
    final Collection<SearchResult> results = new ArrayList<>();
    if (StringUtils.isBlank(context.getAnnotationData())) {
        return results;
    }
    try {
        // parse the annotation data
        final Criteria criteria = CriteriaSerDe.deserialize(context.getAnnotationData());
        // ensure there are some rules
        if (criteria.getRules() != null) {
            final FlowFilePolicy flowFilePolicy = criteria.getFlowFilePolicy();
            if (flowFilePolicy != null && StringUtils.containsIgnoreCase(flowFilePolicy.name(), term)) {
                results.add(new SearchResult.Builder().label("FlowFile policy").match(flowFilePolicy.name()).build());
            }
            for (final Rule rule : criteria.getRules()) {
                if (StringUtils.containsIgnoreCase(rule.getName(), term)) {
                    results.add(new SearchResult.Builder().label("Rule name").match(rule.getName()).build());
                }
                // ensure there are some conditions
                if (rule.getConditions() != null) {
                    for (final Condition condition : rule.getConditions()) {
                        if (StringUtils.containsIgnoreCase(condition.getExpression(), term)) {
                            results.add(new SearchResult.Builder().label(String.format("Condition in rule '%s'", rule.getName())).match(condition.getExpression()).build());
                        }
                    }
                }
                // ensure there are some actions
                if (rule.getActions() != null) {
                    for (final Action action : rule.getActions()) {
                        if (StringUtils.containsIgnoreCase(action.getAttribute(), term)) {
                            results.add(new SearchResult.Builder().label(String.format("Action in rule '%s'", rule.getName())).match(action.getAttribute()).build());
                        }
                        if (StringUtils.containsIgnoreCase(action.getValue(), term)) {
                            results.add(new SearchResult.Builder().label(String.format("Action in rule '%s'", rule.getName())).match(action.getValue()).build());
                        }
                    }
                }
            }
        }
        return results;
    } catch (Exception e) {
        return results;
    }
}

Example 10 with Rule

use of org.apache.nifi.update.attributes.Rule in project nifi by apache.

the class UpdateAttribute method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    criteriaCache.set(CriteriaSerDe.deserialize(context.getAnnotationData()));
    propertyValues.clear();
    if (stateful) {
        StateManager stateManager = context.getStateManager();
        StateMap state = stateManager.getState(Scope.LOCAL);
        HashMap<String, String> tempMap = new HashMap<>();
        tempMap.putAll(state.toMap());
        String initValue = context.getProperty(STATEFUL_VARIABLES_INIT_VALUE).getValue();
        // Initialize the stateful default actions
        for (PropertyDescriptor entry : context.getProperties().keySet()) {
            if (entry.isDynamic()) {
                if (!tempMap.containsKey(entry.getName())) {
                    tempMap.put(entry.getName(), initValue);
                }
            }
        }
        // Initialize the stateful actions if the criteria exists
        final Criteria criteria = criteriaCache.get();
        if (criteria != null) {
            for (Rule rule : criteria.getRules()) {
                for (Action action : rule.getActions()) {
                    if (!tempMap.containsKey(action.getAttribute())) {
                        tempMap.put(action.getAttribute(), initValue);
                    }
                }
            }
        }
        context.getStateManager().setState(tempMap, Scope.LOCAL);
    }
    defaultActions = getDefaultActions(context.getProperties());
    debugEnabled = getLogger().isDebugEnabled();
}