Examples with AtomicDistributedMapCacheClient org.apache.nifi.distributed.cache.client.AtomicDistributedMapCacheClient used on opensource projects

Example 1 with AtomicDistributedMapCacheClient

use of org.apache.nifi.distributed.cache.client.AtomicDistributedMapCacheClient in project nifi by apache.

the class Notify method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    final PropertyValue signalIdProperty = context.getProperty(RELEASE_SIGNAL_IDENTIFIER);
    final PropertyValue counterNameProperty = context.getProperty(SIGNAL_COUNTER_NAME);
    final PropertyValue deltaProperty = context.getProperty(SIGNAL_COUNTER_DELTA);
    final String attributeCacheRegex = context.getProperty(ATTRIBUTE_CACHE_REGEX).getValue();
    final Integer bufferCount = context.getProperty(SIGNAL_BUFFER_COUNT).asInteger();
    // the cache client used to interact with the distributed cache.
    final AtomicDistributedMapCacheClient cache = context.getProperty(DISTRIBUTED_CACHE_SERVICE).asControllerService(AtomicDistributedMapCacheClient.class);
    final WaitNotifyProtocol protocol = new WaitNotifyProtocol(cache);
    final Map<String, SignalBuffer> signalBuffers = new HashMap<>();
    for (int i = 0; i < bufferCount; i++) {
        final FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }
        // Signal id is computed from attribute 'RELEASE_SIGNAL_IDENTIFIER' with expression language support
        final String signalId = signalIdProperty.evaluateAttributeExpressions(flowFile).getValue();
        // if the computed value is null, or empty, we transfer the flow file to failure relationship
        if (StringUtils.isBlank(signalId)) {
            logger.error("FlowFile {} has no attribute for given Release Signal Identifier", new Object[] { flowFile });
            // set 'notified' attribute
            session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(false)), REL_FAILURE);
            continue;
        }
        String counterName = counterNameProperty.evaluateAttributeExpressions(flowFile).getValue();
        if (StringUtils.isEmpty(counterName)) {
            counterName = WaitNotifyProtocol.DEFAULT_COUNT_NAME;
        }
        int delta = 1;
        if (deltaProperty.isSet()) {
            final String deltaStr = deltaProperty.evaluateAttributeExpressions(flowFile).getValue();
            try {
                delta = Integer.parseInt(deltaStr);
            } catch (final NumberFormatException e) {
                logger.error("Failed to calculate delta for FlowFile {} due to {}", new Object[] { flowFile, e }, e);
                session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(false)), REL_FAILURE);
                continue;
            }
        }
        if (!signalBuffers.containsKey(signalId)) {
            signalBuffers.put(signalId, new SignalBuffer());
        }
        final SignalBuffer signalBuffer = signalBuffers.get(signalId);
        if (StringUtils.isNotEmpty(attributeCacheRegex)) {
            flowFile.getAttributes().entrySet().stream().filter(e -> (!e.getKey().equals("uuid") && e.getKey().matches(attributeCacheRegex))).forEach(e -> signalBuffer.attributesToCache.put(e.getKey(), e.getValue()));
        }
        signalBuffer.incrementDelta(counterName, delta);
        signalBuffer.flowFiles.add(flowFile);
        if (logger.isDebugEnabled()) {
            logger.debug("Cached release signal identifier {} counterName {} from FlowFile {}", new Object[] { signalId, counterName, flowFile });
        }
    }
    signalBuffers.forEach((signalId, signalBuffer) -> {
        // retry after yielding for a while.
        try {
            protocol.notify(signalId, signalBuffer.deltas, signalBuffer.attributesToCache);
            signalBuffer.flowFiles.forEach(flowFile -> session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(true)), REL_SUCCESS));
        } catch (IOException e) {
            throw new RuntimeException(String.format("Unable to communicate with cache when processing %s due to %s", signalId, e), e);
        }
    });
}

Example 2 with AtomicDistributedMapCacheClient

use of org.apache.nifi.distributed.cache.client.AtomicDistributedMapCacheClient in project nifi by apache.

the class Wait method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    // Signal id is computed from attribute 'RELEASE_SIGNAL_IDENTIFIER' with expression language support
    final PropertyValue signalIdProperty = context.getProperty(RELEASE_SIGNAL_IDENTIFIER);
    final Integer bufferCount = context.getProperty(WAIT_BUFFER_COUNT).asInteger();
    final Map<Relationship, List<FlowFile>> processedFlowFiles = new HashMap<>();
    final Function<Relationship, List<FlowFile>> getFlowFilesFor = r -> processedFlowFiles.computeIfAbsent(r, k -> new ArrayList<>());
    final AtomicReference<String> targetSignalId = new AtomicReference<>();
    final AtomicInteger bufferedCount = new AtomicInteger(0);
    final List<FlowFile> failedFilteringFlowFiles = new ArrayList<>();
    final Supplier<FlowFileFilter.FlowFileFilterResult> acceptResultSupplier = () -> bufferedCount.incrementAndGet() == bufferCount ? ACCEPT_AND_TERMINATE : ACCEPT_AND_CONTINUE;
    final List<FlowFile> flowFiles = session.get(f -> {
        final String fSignalId = signalIdProperty.evaluateAttributeExpressions(f).getValue();
        // if the computed value is null, or empty, we transfer the FlowFile to failure relationship
        if (StringUtils.isBlank(fSignalId)) {
            // We can't penalize f before getting it from session, so keep it in a temporal list.
            logger.error("FlowFile {} has no attribute for given Release Signal Identifier", new Object[] { f });
            failedFilteringFlowFiles.add(f);
            return ACCEPT_AND_CONTINUE;
        }
        final String targetSignalIdStr = targetSignalId.get();
        if (targetSignalIdStr == null) {
            // This is the first one.
            targetSignalId.set(fSignalId);
            return acceptResultSupplier.get();
        }
        if (targetSignalIdStr.equals(fSignalId)) {
            return acceptResultSupplier.get();
        }
        return REJECT_AND_CONTINUE;
    });
    final String attributeCopyMode = context.getProperty(ATTRIBUTE_COPY_MODE).getValue();
    final boolean replaceOriginalAttributes = ATTRIBUTE_COPY_REPLACE.getValue().equals(attributeCopyMode);
    final AtomicReference<Signal> signalRef = new AtomicReference<>();
    // This map contains original counts before those are consumed to release incoming FlowFiles.
    final HashMap<String, Long> originalSignalCounts = new HashMap<>();
    final Consumer<FlowFile> transferToFailure = flowFile -> {
        flowFile = session.penalize(flowFile);
        getFlowFilesFor.apply(REL_FAILURE).add(flowFile);
    };
    final Consumer<Entry<Relationship, List<FlowFile>>> transferFlowFiles = routedFlowFiles -> {
        Relationship relationship = routedFlowFiles.getKey();
        if (REL_WAIT.equals(relationship)) {
            final String waitMode = context.getProperty(WAIT_MODE).getValue();
            if (WAIT_MODE_KEEP_IN_UPSTREAM.getValue().equals(waitMode)) {
                // Transfer to self.
                relationship = Relationship.SELF;
            }
        }
        final List<FlowFile> flowFilesWithSignalAttributes = routedFlowFiles.getValue().stream().map(f -> copySignalAttributes(session, f, signalRef.get(), originalSignalCounts, replaceOriginalAttributes)).collect(Collectors.toList());
        session.transfer(flowFilesWithSignalAttributes, relationship);
    };
    failedFilteringFlowFiles.forEach(f -> {
        flowFiles.remove(f);
        transferToFailure.accept(f);
    });
    if (flowFiles.isEmpty()) {
        // If there was nothing but failed FlowFiles while filtering, transfer those and end immediately.
        processedFlowFiles.entrySet().forEach(transferFlowFiles);
        return;
    }
    // the cache client used to interact with the distributed cache
    final AtomicDistributedMapCacheClient cache = context.getProperty(DISTRIBUTED_CACHE_SERVICE).asControllerService(AtomicDistributedMapCacheClient.class);
    final WaitNotifyProtocol protocol = new WaitNotifyProtocol(cache);
    final String signalId = targetSignalId.get();
    final Signal signal;
    // get notifying signal
    try {
        signal = protocol.getSignal(signalId);
        if (signal != null) {
            originalSignalCounts.putAll(signal.getCounts());
        }
        signalRef.set(signal);
    } catch (final IOException e) {
        throw new ProcessException(String.format("Failed to get signal for %s due to %s", signalId, e), e);
    }
    String targetCounterName = null;
    long targetCount = 1;
    int releasableFlowFileCount = 1;
    final List<FlowFile> candidates = new ArrayList<>();
    for (FlowFile flowFile : flowFiles) {
        // Set wait start timestamp if it's not set yet
        String waitStartTimestamp = flowFile.getAttribute(WAIT_START_TIMESTAMP);
        if (waitStartTimestamp == null) {
            waitStartTimestamp = String.valueOf(System.currentTimeMillis());
            flowFile = session.putAttribute(flowFile, WAIT_START_TIMESTAMP, waitStartTimestamp);
        }
        long lWaitStartTimestamp;
        try {
            lWaitStartTimestamp = Long.parseLong(waitStartTimestamp);
        } catch (NumberFormatException nfe) {
            logger.error("{} has an invalid value '{}' on FlowFile {}", new Object[] { WAIT_START_TIMESTAMP, waitStartTimestamp, flowFile });
            transferToFailure.accept(flowFile);
            continue;
        }
        // check for expiration
        long expirationDuration = context.getProperty(EXPIRATION_DURATION).asTimePeriod(TimeUnit.MILLISECONDS);
        long now = System.currentTimeMillis();
        if (now > (lWaitStartTimestamp + expirationDuration)) {
            logger.info("FlowFile {} expired after {}ms", new Object[] { flowFile, (now - lWaitStartTimestamp) });
            getFlowFilesFor.apply(REL_EXPIRED).add(flowFile);
            continue;
        }
        // If there's no signal yet, then we don't have to evaluate target counts. Return immediately.
        if (signal == null) {
            if (logger.isDebugEnabled()) {
                logger.debug("No release signal found for {} on FlowFile {} yet", new Object[] { signalId, flowFile });
            }
            getFlowFilesFor.apply(REL_WAIT).add(flowFile);
            continue;
        }
        // Fix target counter name and count from current FlowFile, if those are not set yet.
        if (candidates.isEmpty()) {
            targetCounterName = context.getProperty(SIGNAL_COUNTER_NAME).evaluateAttributeExpressions(flowFile).getValue();
            try {
                targetCount = Long.valueOf(context.getProperty(TARGET_SIGNAL_COUNT).evaluateAttributeExpressions(flowFile).getValue());
            } catch (final NumberFormatException e) {
                transferToFailure.accept(flowFile);
                logger.error("Failed to parse targetCount when processing {} due to {}", new Object[] { flowFile, e }, e);
                continue;
            }
            try {
                releasableFlowFileCount = Integer.valueOf(context.getProperty(RELEASABLE_FLOWFILE_COUNT).evaluateAttributeExpressions(flowFile).getValue());
            } catch (final NumberFormatException e) {
                transferToFailure.accept(flowFile);
                logger.error("Failed to parse releasableFlowFileCount when processing {} due to {}", new Object[] { flowFile, e }, e);
                continue;
            }
        }
        // FlowFile is now validated and added to candidates.
        candidates.add(flowFile);
    }
    boolean waitCompleted = false;
    boolean waitProgressed = false;
    if (signal != null && !candidates.isEmpty()) {
        if (releasableFlowFileCount > 0) {
            signal.releaseCandidates(targetCounterName, targetCount, releasableFlowFileCount, candidates, released -> getFlowFilesFor.apply(REL_SUCCESS).addAll(released), waiting -> getFlowFilesFor.apply(REL_WAIT).addAll(waiting));
            waitCompleted = signal.getTotalCount() == 0 && signal.getReleasableCount() == 0;
            waitProgressed = !getFlowFilesFor.apply(REL_SUCCESS).isEmpty();
        } else {
            boolean reachedTargetCount = StringUtils.isBlank(targetCounterName) ? signal.isTotalCountReached(targetCount) : signal.isCountReached(targetCounterName, targetCount);
            if (reachedTargetCount) {
                getFlowFilesFor.apply(REL_SUCCESS).addAll(candidates);
            } else {
                getFlowFilesFor.apply(REL_WAIT).addAll(candidates);
            }
        }
    }
    // Transfer FlowFiles.
    processedFlowFiles.entrySet().forEach(transferFlowFiles);
    // Update signal if needed.
    try {
        if (waitCompleted) {
            protocol.complete(signalId);
        } else if (waitProgressed) {
            protocol.replace(signal);
        }
    } catch (final IOException e) {
        session.rollback();
        throw new ProcessException(String.format("Unable to communicate with cache while updating %s due to %s", signalId, e), e);
    }
}