Search in sources :

Example 71 with ProcessContext

use of org.apache.nifi.processor.ProcessContext in project nifi by apache.

the class AbstractListProcessor method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    Long minTimestampToListMillis = lastListedLatestEntryTimestampMillis;
    if (this.lastListedLatestEntryTimestampMillis == null || this.lastProcessedLatestEntryTimestampMillis == null || justElectedPrimaryNode) {
        try {
            // Attempt to retrieve state from the state manager if a last listing was not yet established or
            // if just elected the primary node
            final StateMap stateMap = context.getStateManager().getState(getStateScope(context));
            latestIdentifiersProcessed.clear();
            for (Map.Entry<String, String> state : stateMap.toMap().entrySet()) {
                final String k = state.getKey();
                final String v = state.getValue();
                if (v == null || v.isEmpty()) {
                    continue;
                }
                if (LATEST_LISTED_ENTRY_TIMESTAMP_KEY.equals(k)) {
                    minTimestampToListMillis = Long.parseLong(v);
                    // If our determined timestamp is the same as that of our last listing, skip this execution as there are no updates
                    if (minTimestampToListMillis.equals(this.lastListedLatestEntryTimestampMillis)) {
                        context.yield();
                        return;
                    } else {
                        this.lastListedLatestEntryTimestampMillis = minTimestampToListMillis;
                    }
                } else if (LAST_PROCESSED_LATEST_ENTRY_TIMESTAMP_KEY.equals(k)) {
                    this.lastProcessedLatestEntryTimestampMillis = Long.parseLong(v);
                } else if (k.startsWith(IDENTIFIER_PREFIX)) {
                    latestIdentifiersProcessed.add(v);
                }
            }
            justElectedPrimaryNode = false;
        } catch (final IOException ioe) {
            getLogger().error("Failed to retrieve timestamp of last listing from the State Manager. Will not perform listing until this is accomplished.");
            context.yield();
            return;
        }
    }
    final List<T> entityList;
    final long currentRunTimeNanos = System.nanoTime();
    final long currentRunTimeMillis = System.currentTimeMillis();
    try {
        // track of when this last executed for consideration of the lag nanos
        entityList = performListing(context, minTimestampToListMillis);
    } catch (final IOException e) {
        getLogger().error("Failed to perform listing on remote host due to {}", e);
        context.yield();
        return;
    }
    if (entityList == null || entityList.isEmpty()) {
        context.yield();
        return;
    }
    Long latestListedEntryTimestampThisCycleMillis = null;
    final TreeMap<Long, List<T>> orderedEntries = new TreeMap<>();
    // Build a sorted map to determine the latest possible entries
    boolean targetSystemHasMilliseconds = false;
    boolean targetSystemHasSeconds = false;
    for (final T entity : entityList) {
        final long entityTimestampMillis = entity.getTimestamp();
        if (!targetSystemHasMilliseconds) {
            targetSystemHasMilliseconds = entityTimestampMillis % 1000 > 0;
        }
        if (!targetSystemHasSeconds) {
            targetSystemHasSeconds = entityTimestampMillis % 60_000 > 0;
        }
        // New entries are all those that occur at or after the associated timestamp
        final boolean newEntry = minTimestampToListMillis == null || entityTimestampMillis >= minTimestampToListMillis && entityTimestampMillis >= lastProcessedLatestEntryTimestampMillis;
        if (newEntry) {
            List<T> entitiesForTimestamp = orderedEntries.get(entity.getTimestamp());
            if (entitiesForTimestamp == null) {
                entitiesForTimestamp = new ArrayList<T>();
                orderedEntries.put(entity.getTimestamp(), entitiesForTimestamp);
            }
            entitiesForTimestamp.add(entity);
        }
    }
    int flowfilesCreated = 0;
    if (orderedEntries.size() > 0) {
        latestListedEntryTimestampThisCycleMillis = orderedEntries.lastKey();
        // Determine target system time precision.
        String specifiedPrecision = context.getProperty(TARGET_SYSTEM_TIMESTAMP_PRECISION).getValue();
        if (StringUtils.isBlank(specifiedPrecision)) {
            // If TARGET_SYSTEM_TIMESTAMP_PRECISION is not supported by the Processor, then specifiedPrecision can be null, instead of its default value.
            specifiedPrecision = getDefaultTimePrecision();
        }
        final TimeUnit targetSystemTimePrecision = PRECISION_AUTO_DETECT.getValue().equals(specifiedPrecision) ? targetSystemHasMilliseconds ? TimeUnit.MILLISECONDS : targetSystemHasSeconds ? TimeUnit.SECONDS : TimeUnit.MINUTES : PRECISION_MILLIS.getValue().equals(specifiedPrecision) ? TimeUnit.MILLISECONDS : PRECISION_SECONDS.getValue().equals(specifiedPrecision) ? TimeUnit.SECONDS : TimeUnit.MINUTES;
        final Long listingLagMillis = LISTING_LAG_MILLIS.get(targetSystemTimePrecision);
        // another iteration has occurred without new files and special handling is needed to avoid starvation
        if (latestListedEntryTimestampThisCycleMillis.equals(lastListedLatestEntryTimestampMillis)) {
            /* We need to wait for another cycle when either:
                 *   - If we have not eclipsed the minimal listing lag needed due to being triggered too soon after the last run
                 *   - The latest listed entity timestamp is equal to the last processed time, meaning we handled those items originally passed over. No need to process it again.
                 */
            final long listingLagNanos = TimeUnit.MILLISECONDS.toNanos(listingLagMillis);
            if (currentRunTimeNanos - lastRunTimeNanos < listingLagNanos || (latestListedEntryTimestampThisCycleMillis.equals(lastProcessedLatestEntryTimestampMillis) && orderedEntries.get(latestListedEntryTimestampThisCycleMillis).stream().allMatch(entity -> latestIdentifiersProcessed.contains(entity.getIdentifier())))) {
                context.yield();
                return;
            }
        } else {
            // Convert minimum reliable timestamp into target system time unit, in order to truncate unreliable digits.
            final long minimumReliableTimestampInFilesystemTimeUnit = targetSystemTimePrecision.convert(currentRunTimeMillis - listingLagMillis, TimeUnit.MILLISECONDS);
            final long minimumReliableTimestampMillis = targetSystemTimePrecision.toMillis(minimumReliableTimestampInFilesystemTimeUnit);
            // The minimum timestamp should be reliable to determine that no further entries will be added with the same timestamp based on the target system time precision.
            if (minimumReliableTimestampMillis < latestListedEntryTimestampThisCycleMillis) {
                // Otherwise, newest entries are held back one cycle to avoid issues in writes occurring exactly when the listing is being performed to avoid missing data
                orderedEntries.remove(latestListedEntryTimestampThisCycleMillis);
            }
        }
        for (Map.Entry<Long, List<T>> timestampEntities : orderedEntries.entrySet()) {
            List<T> entities = timestampEntities.getValue();
            if (timestampEntities.getKey().equals(lastProcessedLatestEntryTimestampMillis)) {
                // Filter out previously processed entities.
                entities = entities.stream().filter(entity -> !latestIdentifiersProcessed.contains(entity.getIdentifier())).collect(Collectors.toList());
            }
            for (T entity : entities) {
                // Create the FlowFile for this path.
                final Map<String, String> attributes = createAttributes(entity, context);
                FlowFile flowFile = session.create();
                flowFile = session.putAllAttributes(flowFile, attributes);
                session.transfer(flowFile, REL_SUCCESS);
                flowfilesCreated++;
            }
        }
    }
    // As long as we have a listing timestamp, there is meaningful state to capture regardless of any outputs generated
    if (latestListedEntryTimestampThisCycleMillis != null) {
        boolean processedNewFiles = flowfilesCreated > 0;
        if (processedNewFiles) {
            // because latestListedEntryTimestampThisCycleMillis might be removed if it's not old enough.
            if (!orderedEntries.lastKey().equals(lastProcessedLatestEntryTimestampMillis)) {
                // If the latest timestamp at this cycle becomes different than the previous one, we need to clear identifiers.
                // If it didn't change, we need to add identifiers.
                latestIdentifiersProcessed.clear();
            }
            // Capture latestIdentifierProcessed.
            latestIdentifiersProcessed.addAll(orderedEntries.lastEntry().getValue().stream().map(T::getIdentifier).collect(Collectors.toList()));
            lastProcessedLatestEntryTimestampMillis = orderedEntries.lastKey();
            getLogger().info("Successfully created listing with {} new objects", new Object[] { flowfilesCreated });
            session.commit();
        }
        lastRunTimeNanos = currentRunTimeNanos;
        if (!latestListedEntryTimestampThisCycleMillis.equals(lastListedLatestEntryTimestampMillis) || processedNewFiles) {
            // the distributed state cache, the node can continue to run (if it is primary node).
            try {
                lastListedLatestEntryTimestampMillis = latestListedEntryTimestampThisCycleMillis;
                persist(latestListedEntryTimestampThisCycleMillis, lastProcessedLatestEntryTimestampMillis, latestIdentifiersProcessed, context.getStateManager(), getStateScope(context));
            } catch (final IOException ioe) {
                getLogger().warn("Unable to save state due to {}. If NiFi is restarted before state is saved, or " + "if another node begins executing this Processor, data duplication may occur.", ioe);
            }
        }
    } else {
        getLogger().debug("There is no data to list. Yielding.");
        context.yield();
        // lastListingTime = 0 so that we don't continually poll the distributed cache / local file system
        if (lastListedLatestEntryTimestampMillis == null) {
            lastListedLatestEntryTimestampMillis = 0L;
        }
        return;
    }
}
Also used : Serializer(org.apache.nifi.distributed.cache.client.Serializer) HashMap(java.util.HashMap) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ProcessException(org.apache.nifi.processor.exception.ProcessException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PrimaryNodeState(org.apache.nifi.annotation.notification.PrimaryNodeState) Scope(org.apache.nifi.components.state.Scope) Relationship(org.apache.nifi.processor.Relationship) Map(java.util.Map) DistributedMapCacheClient(org.apache.nifi.distributed.cache.client.DistributedMapCacheClient) JsonParseException(com.fasterxml.jackson.core.JsonParseException) TriggerSerially(org.apache.nifi.annotation.behavior.TriggerSerially) OutputStream(java.io.OutputStream) Properties(java.util.Properties) FlowFile(org.apache.nifi.flowfile.FlowFile) StateManager(org.apache.nifi.components.state.StateManager) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) ProcessSession(org.apache.nifi.processor.ProcessSession) IOException(java.io.IOException) StringUtils(org.apache.nifi.util.StringUtils) FileInputStream(java.io.FileInputStream) AllowableValue(org.apache.nifi.components.AllowableValue) Collectors(java.util.stream.Collectors) StateMap(org.apache.nifi.components.state.StateMap) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) Deserializer(org.apache.nifi.distributed.cache.client.Deserializer) TimeUnit(java.util.concurrent.TimeUnit) Stateful(org.apache.nifi.annotation.behavior.Stateful) OnScheduled(org.apache.nifi.annotation.lifecycle.OnScheduled) List(java.util.List) TreeMap(java.util.TreeMap) OnPrimaryNodeStateChange(org.apache.nifi.annotation.notification.OnPrimaryNodeStateChange) DeserializationException(org.apache.nifi.distributed.cache.client.exception.DeserializationException) SerializationException(org.apache.nifi.distributed.cache.client.exception.SerializationException) AbstractProcessor(org.apache.nifi.processor.AbstractProcessor) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) Collections(java.util.Collections) FlowFile(org.apache.nifi.flowfile.FlowFile) StateMap(org.apache.nifi.components.state.StateMap) IOException(java.io.IOException) TreeMap(java.util.TreeMap) TimeUnit(java.util.concurrent.TimeUnit) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) StateMap(org.apache.nifi.components.state.StateMap) TreeMap(java.util.TreeMap)

Example 72 with ProcessContext

use of org.apache.nifi.processor.ProcessContext in project nifi by apache.

the class ListFile method createFileFilter.

private BiPredicate<Path, BasicFileAttributes> createFileFilter(final ProcessContext context) {
    final long minSize = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue();
    final Double maxSize = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B);
    final long minAge = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
    final Long maxAge = context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
    final boolean ignoreHidden = context.getProperty(IGNORE_HIDDEN_FILES).asBoolean();
    final Pattern filePattern = Pattern.compile(context.getProperty(FILE_FILTER).getValue());
    final String indir = context.getProperty(DIRECTORY).evaluateAttributeExpressions().getValue();
    final boolean recurseDirs = context.getProperty(RECURSE).asBoolean();
    final String pathPatternStr = context.getProperty(PATH_FILTER).getValue();
    final Pattern pathPattern = (!recurseDirs || pathPatternStr == null) ? null : Pattern.compile(pathPatternStr);
    return (path, attributes) -> {
        if (minSize > attributes.size()) {
            return false;
        }
        if (maxSize != null && maxSize < attributes.size()) {
            return false;
        }
        final long fileAge = System.currentTimeMillis() - attributes.lastModifiedTime().toMillis();
        if (minAge > fileAge) {
            return false;
        }
        if (maxAge != null && maxAge < fileAge) {
            return false;
        }
        if (ignoreHidden && path.toFile().isHidden()) {
            return false;
        }
        if (pathPattern != null) {
            Path reldir = Paths.get(indir).relativize(path).getParent();
            if (reldir != null && !reldir.toString().isEmpty()) {
                if (!pathPattern.matcher(reldir.toString()).matches()) {
                    return false;
                }
            }
        }
        // Verify that we have at least read permissions on the file we're considering grabbing
        if (!Files.isReadable(path)) {
            return false;
        }
        return filePattern.matcher(path.getFileName().toString()).matches();
    };
}
Also used : StandardValidators(org.apache.nifi.processor.util.StandardValidators) Date(java.util.Date) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) Scope(org.apache.nifi.components.state.Scope) Locale(java.util.Locale) BasicFileAttributeView(java.nio.file.attribute.BasicFileAttributeView) Map(java.util.Map) Path(java.nio.file.Path) TriggerSerially(org.apache.nifi.annotation.behavior.TriggerSerially) DateFormat(java.text.DateFormat) Set(java.util.Set) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) AllowableValue(org.apache.nifi.components.AllowableValue) FileOwnerAttributeView(java.nio.file.attribute.FileOwnerAttributeView) Collectors(java.util.stream.Collectors) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) Stateful(org.apache.nifi.annotation.behavior.Stateful) AbstractListProcessor(org.apache.nifi.processor.util.list.AbstractListProcessor) List(java.util.List) Stream(java.util.stream.Stream) Tags(org.apache.nifi.annotation.documentation.Tags) DataUnit(org.apache.nifi.processor.DataUnit) Pattern(java.util.regex.Pattern) ProcessorInitializationContext(org.apache.nifi.processor.ProcessorInitializationContext) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) FileInfo(org.apache.nifi.processors.standard.util.FileInfo) PosixFileAttributeView(java.nio.file.attribute.PosixFileAttributeView) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) BiPredicate(java.util.function.BiPredicate) PosixFilePermissions(java.nio.file.attribute.PosixFilePermissions) Relationship(org.apache.nifi.processor.Relationship) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) FileStore(java.nio.file.FileStore) Files(java.nio.file.Files) ProcessContext(org.apache.nifi.processor.ProcessContext) IOException(java.io.IOException) SeeAlso(org.apache.nifi.annotation.documentation.SeeAlso) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) OnScheduled(org.apache.nifi.annotation.lifecycle.OnScheduled) Paths(java.nio.file.Paths) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) Collections(java.util.Collections) Path(java.nio.file.Path) Pattern(java.util.regex.Pattern)

Example 73 with ProcessContext

use of org.apache.nifi.processor.ProcessContext in project nifi by apache.

the class LogAttribute method getAttributesToLog.

private Set<String> getAttributesToLog(final Set<String> flowFileAttrKeys, final ProcessContext context) {
    // collect properties
    final String attrsToLogValue = context.getProperty(ATTRIBUTES_TO_LOG_CSV).getValue();
    final String attrsToRemoveValue = context.getProperty(ATTRIBUTES_TO_IGNORE_CSV).getValue();
    final Set<String> attrsToLog = StringUtils.isBlank(attrsToLogValue) ? Sets.newHashSet(flowFileAttrKeys) : Sets.newHashSet(attrsToLogValue.split("\\s*,\\s*"));
    final Set<String> attrsToRemove = StringUtils.isBlank(attrsToRemoveValue) ? Sets.newHashSet() : Sets.newHashSet(attrsToRemoveValue.split("\\s*,\\s*"));
    final Pattern attrsToLogRegex = Pattern.compile(context.getProperty(ATTRIBUTES_TO_LOG_REGEX).getValue());
    final String attrsToRemoveRegexValue = context.getProperty(ATTRIBUTES_TO_IGNORE_REGEX).getValue();
    final Pattern attrsToRemoveRegex = attrsToRemoveRegexValue == null ? null : Pattern.compile(context.getProperty(ATTRIBUTES_TO_IGNORE_REGEX).getValue());
    return flowFileAttrKeys.stream().filter(candidate -> {
        // if this property was configured to be logged, or if the regular expression of properties to log matches
        if ((attrsToLog.isEmpty() || attrsToLog.contains(candidate)) && attrsToLogRegex.matcher(candidate).matches()) {
            // log properties we've _not_ configured either explicitly or by regular expression to be ignored.
            if ((attrsToRemove.isEmpty() || !attrsToRemove.contains(candidate)) && (attrsToRemoveRegex == null || !attrsToRemoveRegex.matcher(candidate).matches())) {
                return true;
            }
        }
        return false;
    }).collect(Collectors.toCollection(TreeSet::new));
}
Also used : StandardValidators(org.apache.nifi.processor.util.StandardValidators) Date(java.util.Date) EventDriven(org.apache.nifi.annotation.behavior.EventDriven) ComponentLog(org.apache.nifi.logging.ComponentLog) StringUtils(org.apache.commons.lang3.StringUtils) SideEffectFree(org.apache.nifi.annotation.behavior.SideEffectFree) TreeSet(java.util.TreeSet) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ProcessException(org.apache.nifi.processor.exception.ProcessException) StringUtil(org.eclipse.jetty.util.StringUtil) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Charset(java.nio.charset.Charset) Relationship(org.apache.nifi.processor.Relationship) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) InputStreamCallback(org.apache.nifi.processor.io.InputStreamCallback) FlowFile(org.apache.nifi.flowfile.FlowFile) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) IOException(java.io.IOException) ProcessSession(org.apache.nifi.processor.ProcessSession) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) SupportsBatching(org.apache.nifi.annotation.behavior.SupportsBatching) AbstractProcessor(org.apache.nifi.processor.AbstractProcessor) Pattern(java.util.regex.Pattern) Tags(org.apache.nifi.annotation.documentation.Tags) Collections(java.util.Collections) ProcessorInitializationContext(org.apache.nifi.processor.ProcessorInitializationContext) InputStream(java.io.InputStream) Pattern(java.util.regex.Pattern)

Example 74 with ProcessContext

use of org.apache.nifi.processor.ProcessContext in project nifi by apache.

the class LookupRecord method route.

@Override
protected Set<Relationship> route(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context, final Tuple<Map<String, RecordPath>, RecordPath> flowFileContext) {
    final Map<String, RecordPath> recordPaths = flowFileContext.getKey();
    final Map<String, Object> lookupCoordinates = new HashMap<>(recordPaths.size());
    for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
        final String coordinateKey = entry.getKey();
        final RecordPath recordPath = entry.getValue();
        final RecordPathResult pathResult = recordPath.evaluate(record);
        final List<FieldValue> lookupFieldValues = pathResult.getSelectedFields().filter(fieldVal -> fieldVal.getValue() != null).collect(Collectors.toList());
        if (lookupFieldValues.isEmpty()) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' did not match any fields in a record for {}; routing record to {}", new Object[] { coordinateKey, flowFile, rels });
            return rels;
        }
        if (lookupFieldValues.size() > 1) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' matched {} fields in a record for {}; routing record to {}", new Object[] { coordinateKey, lookupFieldValues.size(), flowFile, rels });
            return rels;
        }
        final FieldValue fieldValue = lookupFieldValues.get(0);
        final Object coordinateValue = (fieldValue.getValue() instanceof Number || fieldValue.getValue() instanceof Boolean) ? fieldValue.getValue() : DataTypeUtils.toString(fieldValue.getValue(), (String) null);
        lookupCoordinates.put(coordinateKey, coordinateValue);
    }
    final Optional<?> lookupValueOption;
    try {
        lookupValueOption = lookupService.lookup(lookupCoordinates);
    } catch (final Exception e) {
        throw new ProcessException("Failed to lookup coordinates " + lookupCoordinates + " in Lookup Service", e);
    }
    if (!lookupValueOption.isPresent()) {
        final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
        return rels;
    }
    // Ensure that the Record has the appropriate schema to account for the newly added values
    final RecordPath resultPath = flowFileContext.getValue();
    if (resultPath != null) {
        record.incorporateSchema(writeSchema);
        final Object lookupValue = lookupValueOption.get();
        final RecordPathResult resultPathResult = flowFileContext.getValue().evaluate(record);
        final String resultContentsValue = context.getProperty(RESULT_CONTENTS).getValue();
        if (RESULT_RECORD_FIELDS.getValue().equals(resultContentsValue) && lookupValue instanceof Record) {
            final Record lookupRecord = (Record) lookupValue;
            // Use wants to add all fields of the resultant Record to the specified Record Path.
            // If the destination Record Path returns to us a Record, then we will add all field values of
            // the Lookup Record to the destination Record. However, if the destination Record Path returns
            // something other than a Record, then we can't add the fields to it. We can only replace it,
            // because it doesn't make sense to add fields to anything but a Record.
            resultPathResult.getSelectedFields().forEach(fieldVal -> {
                final Object destinationValue = fieldVal.getValue();
                if (destinationValue instanceof Record) {
                    final Record destinationRecord = (Record) destinationValue;
                    for (final String fieldName : lookupRecord.getRawFieldNames()) {
                        final Object value = lookupRecord.getValue(fieldName);
                        destinationRecord.setValue(fieldName, value);
                    }
                } else {
                    final Optional<Record> parentOption = fieldVal.getParentRecord();
                    if (parentOption.isPresent()) {
                        parentOption.get().setValue(fieldVal.getField().getFieldName(), lookupRecord);
                    }
                }
            });
        } else {
            resultPathResult.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(lookupValue));
        }
    }
    final Set<Relationship> rels = routeToMatchedUnmatched ? MATCHED_COLLECTION : SUCCESS_COLLECTION;
    return rels;
}
Also used : Arrays(java.util.Arrays) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) ValidationContext(org.apache.nifi.components.ValidationContext) HashMap(java.util.HashMap) EventDriven(org.apache.nifi.annotation.behavior.EventDriven) SideEffectFree(org.apache.nifi.annotation.behavior.SideEffectFree) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ProcessException(org.apache.nifi.processor.exception.ProcessException) RecordPath(org.apache.nifi.record.path.RecordPath) ArrayList(java.util.ArrayList) RecordPathValidator(org.apache.nifi.record.path.validation.RecordPathValidator) HashSet(java.util.HashSet) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) Relationship(org.apache.nifi.processor.Relationship) Map(java.util.Map) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) ValidationResult(org.apache.nifi.components.ValidationResult) Record(org.apache.nifi.serialization.record.Record) RecordPathResult(org.apache.nifi.record.path.RecordPathResult) FlowFile(org.apache.nifi.flowfile.FlowFile) Collection(java.util.Collection) DataTypeUtils(org.apache.nifi.serialization.record.util.DataTypeUtils) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) SeeAlso(org.apache.nifi.annotation.documentation.SeeAlso) AllowableValue(org.apache.nifi.components.AllowableValue) Collectors(java.util.stream.Collectors) List(java.util.List) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) OnScheduled(org.apache.nifi.annotation.lifecycle.OnScheduled) DynamicProperty(org.apache.nifi.annotation.behavior.DynamicProperty) SupportsBatching(org.apache.nifi.annotation.behavior.SupportsBatching) LookupService(org.apache.nifi.lookup.LookupService) Tuple(org.apache.nifi.util.Tuple) Optional(java.util.Optional) RecordPathCache(org.apache.nifi.record.path.util.RecordPathCache) Tags(org.apache.nifi.annotation.documentation.Tags) FieldValue(org.apache.nifi.record.path.FieldValue) Collections(java.util.Collections) HashMap(java.util.HashMap) RecordPathResult(org.apache.nifi.record.path.RecordPathResult) RecordPath(org.apache.nifi.record.path.RecordPath) ProcessException(org.apache.nifi.processor.exception.ProcessException) ProcessException(org.apache.nifi.processor.exception.ProcessException) Relationship(org.apache.nifi.processor.Relationship) Record(org.apache.nifi.serialization.record.Record) FieldValue(org.apache.nifi.record.path.FieldValue) HashMap(java.util.HashMap) Map(java.util.Map)

Example 75 with ProcessContext

use of org.apache.nifi.processor.ProcessContext in project nifi by apache.

the class MonitorActivity method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final long thresholdMillis = context.getProperty(THRESHOLD).asTimePeriod(TimeUnit.MILLISECONDS);
    final long now = System.currentTimeMillis();
    final ComponentLog logger = getLogger();
    final boolean copyAttributes = context.getProperty(COPY_ATTRIBUTES).asBoolean();
    final boolean isClusterScope = isClusterScope(context, false);
    final boolean shouldReportOnlyOnPrimary = shouldReportOnlyOnPrimary(isClusterScope, context);
    final List<FlowFile> flowFiles = session.get(50);
    boolean isInactive = false;
    long updatedLatestSuccessTransfer = -1;
    StateMap clusterState = null;
    if (flowFiles.isEmpty()) {
        final long previousSuccessMillis = latestSuccessTransfer.get();
        boolean sendInactiveMarker = false;
        isInactive = (now >= previousSuccessMillis + thresholdMillis);
        logger.debug("isInactive={}, previousSuccessMillis={}, now={}", new Object[] { isInactive, previousSuccessMillis, now });
        if (isInactive && isClusterScope) {
            // However, if this node is active, we don't have to look at cluster state.
            try {
                clusterState = context.getStateManager().getState(Scope.CLUSTER);
                if (clusterState != null && !StringUtils.isEmpty(clusterState.get(STATE_KEY_LATEST_SUCCESS_TRANSFER))) {
                    final long latestReportedClusterActivity = Long.valueOf(clusterState.get(STATE_KEY_LATEST_SUCCESS_TRANSFER));
                    isInactive = (now >= latestReportedClusterActivity + thresholdMillis);
                    if (!isInactive) {
                        // This node has been inactive, but other node has more recent activity.
                        updatedLatestSuccessTransfer = latestReportedClusterActivity;
                    }
                    logger.debug("isInactive={}, latestReportedClusterActivity={}", new Object[] { isInactive, latestReportedClusterActivity });
                }
            } catch (IOException e) {
                logger.error("Failed to access cluster state. Activity will not be monitored properly until this is addressed.", e);
            }
        }
        if (isInactive) {
            final boolean continual = context.getProperty(CONTINUALLY_SEND_MESSAGES).asBoolean();
            sendInactiveMarker = !inactive.getAndSet(true) || (continual && (now > lastInactiveMessage.get() + thresholdMillis));
        }
        if (sendInactiveMarker && shouldThisNodeReport(isClusterScope, shouldReportOnlyOnPrimary)) {
            lastInactiveMessage.set(System.currentTimeMillis());
            FlowFile inactiveFlowFile = session.create();
            inactiveFlowFile = session.putAttribute(inactiveFlowFile, "inactivityStartMillis", String.valueOf(previousSuccessMillis));
            inactiveFlowFile = session.putAttribute(inactiveFlowFile, "inactivityDurationMillis", String.valueOf(now - previousSuccessMillis));
            final byte[] outBytes = context.getProperty(INACTIVITY_MESSAGE).evaluateAttributeExpressions(inactiveFlowFile).getValue().getBytes(UTF8);
            inactiveFlowFile = session.write(inactiveFlowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream out) throws IOException {
                    out.write(outBytes);
                }
            });
            session.getProvenanceReporter().create(inactiveFlowFile);
            session.transfer(inactiveFlowFile, REL_INACTIVE);
            logger.info("Transferred {} to 'inactive'", new Object[] { inactiveFlowFile });
        } else {
            // no need to dominate CPU checking times; let other processors run for a bit.
            context.yield();
        }
    } else {
        session.transfer(flowFiles, REL_SUCCESS);
        updatedLatestSuccessTransfer = now;
        logger.info("Transferred {} FlowFiles to 'success'", new Object[] { flowFiles.size() });
        final long latestStateReportTimestamp = latestReportedNodeState.get();
        if (isClusterScope && (now - latestStateReportTimestamp) > (thresholdMillis / 3)) {
            // We don't want to hit the state manager every onTrigger(), but often enough to detect activeness.
            try {
                final StateManager stateManager = context.getStateManager();
                final StateMap state = stateManager.getState(Scope.CLUSTER);
                final Map<String, String> newValues = new HashMap<>();
                // Persist attributes so that other nodes can copy it
                if (copyAttributes) {
                    newValues.putAll(flowFiles.get(0).getAttributes());
                }
                newValues.put(STATE_KEY_LATEST_SUCCESS_TRANSFER, String.valueOf(now));
                if (state == null || state.getVersion() == -1) {
                    stateManager.setState(newValues, Scope.CLUSTER);
                } else {
                    final String existingTimestamp = state.get(STATE_KEY_LATEST_SUCCESS_TRANSFER);
                    if (StringUtils.isEmpty(existingTimestamp) || Long.parseLong(existingTimestamp) < now) {
                        // If this returns false due to race condition, it's not a problem since we just need
                        // the latest active timestamp.
                        stateManager.replace(state, newValues, Scope.CLUSTER);
                    } else {
                        logger.debug("Existing state has more recent timestamp, didn't update state.");
                    }
                }
                latestReportedNodeState.set(now);
            } catch (IOException e) {
                logger.error("Failed to access cluster state. Activity will not be monitored properly until this is addressed.", e);
            }
        }
    }
    if (!isInactive) {
        final long inactivityStartMillis = latestSuccessTransfer.get();
        if (updatedLatestSuccessTransfer > -1) {
            latestSuccessTransfer.set(updatedLatestSuccessTransfer);
        }
        if (inactive.getAndSet(false) && shouldThisNodeReport(isClusterScope, shouldReportOnlyOnPrimary)) {
            FlowFile activityRestoredFlowFile = session.create();
            if (copyAttributes) {
                final Map<String, String> attributes = new HashMap<>();
                if (flowFiles.size() > 0) {
                    // copy attributes from the first flow file in the list
                    attributes.putAll(flowFiles.get(0).getAttributes());
                } else if (clusterState != null) {
                    attributes.putAll(clusterState.toMap());
                    attributes.remove(STATE_KEY_LATEST_SUCCESS_TRANSFER);
                }
                // don't copy the UUID
                attributes.remove(CoreAttributes.UUID.key());
                activityRestoredFlowFile = session.putAllAttributes(activityRestoredFlowFile, attributes);
            }
            activityRestoredFlowFile = session.putAttribute(activityRestoredFlowFile, "inactivityStartMillis", String.valueOf(inactivityStartMillis));
            activityRestoredFlowFile = session.putAttribute(activityRestoredFlowFile, "inactivityDurationMillis", String.valueOf(now - inactivityStartMillis));
            final byte[] outBytes = context.getProperty(ACTIVITY_RESTORED_MESSAGE).evaluateAttributeExpressions(activityRestoredFlowFile).getValue().getBytes(UTF8);
            activityRestoredFlowFile = session.write(activityRestoredFlowFile, out -> out.write(outBytes));
            session.getProvenanceReporter().create(activityRestoredFlowFile);
            session.transfer(activityRestoredFlowFile, REL_ACTIVITY_RESTORED);
            logger.info("Transferred {} to 'activity.restored'", new Object[] { activityRestoredFlowFile });
        }
    }
}
Also used : OutputStreamCallback(org.apache.nifi.processor.io.OutputStreamCallback) StandardValidators(org.apache.nifi.processor.util.StandardValidators) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) ComponentLog(org.apache.nifi.logging.ComponentLog) SideEffectFree(org.apache.nifi.annotation.behavior.SideEffectFree) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Charset(java.nio.charset.Charset) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) Scope(org.apache.nifi.components.state.Scope) Relationship(org.apache.nifi.processor.Relationship) Map(java.util.Map) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) TriggerSerially(org.apache.nifi.annotation.behavior.TriggerSerially) ValidationResult(org.apache.nifi.components.ValidationResult) OutputStream(java.io.OutputStream) TriggerWhenEmpty(org.apache.nifi.annotation.behavior.TriggerWhenEmpty) FlowFile(org.apache.nifi.flowfile.FlowFile) StateManager(org.apache.nifi.components.state.StateManager) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) IOException(java.io.IOException) ProcessSession(org.apache.nifi.processor.ProcessSession) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) StringUtils(org.apache.nifi.util.StringUtils) AllowableValue(org.apache.nifi.components.AllowableValue) StateMap(org.apache.nifi.components.state.StateMap) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) Stateful(org.apache.nifi.annotation.behavior.Stateful) OnScheduled(org.apache.nifi.annotation.lifecycle.OnScheduled) AbstractProcessor(org.apache.nifi.processor.AbstractProcessor) Tags(org.apache.nifi.annotation.documentation.Tags) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) Collections(java.util.Collections) OnStopped(org.apache.nifi.annotation.lifecycle.OnStopped) ProcessorInitializationContext(org.apache.nifi.processor.ProcessorInitializationContext) FlowFile(org.apache.nifi.flowfile.FlowFile) HashMap(java.util.HashMap) StateMap(org.apache.nifi.components.state.StateMap) OutputStream(java.io.OutputStream) IOException(java.io.IOException) ComponentLog(org.apache.nifi.logging.ComponentLog) StateManager(org.apache.nifi.components.state.StateManager) OutputStreamCallback(org.apache.nifi.processor.io.OutputStreamCallback)

Aggregations

ProcessContext (org.apache.nifi.processor.ProcessContext)115 Test (org.junit.Test)67 TestRunner (org.apache.nifi.util.TestRunner)56 ProcessSession (org.apache.nifi.processor.ProcessSession)49 FlowFile (org.apache.nifi.flowfile.FlowFile)40 MockFlowFile (org.apache.nifi.util.MockFlowFile)39 HashSet (java.util.HashSet)35 Relationship (org.apache.nifi.processor.Relationship)35 List (java.util.List)34 PropertyDescriptor (org.apache.nifi.components.PropertyDescriptor)34 ArrayList (java.util.ArrayList)33 Set (java.util.Set)33 Tags (org.apache.nifi.annotation.documentation.Tags)31 IOException (java.io.IOException)30 HashMap (java.util.HashMap)30 CapabilityDescription (org.apache.nifi.annotation.documentation.CapabilityDescription)30 ProcessException (org.apache.nifi.processor.exception.ProcessException)30 Collections (java.util.Collections)29 InputRequirement (org.apache.nifi.annotation.behavior.InputRequirement)29 ProcessSessionFactory (org.apache.nifi.processor.ProcessSessionFactory)29