Examples with StandardProcessSession org.apache.nifi.controller.repository.StandardProcessSession used on opensource projects

Example 1 with StandardProcessSession

use of org.apache.nifi.controller.repository.StandardProcessSession in project nifi by apache.

the class ExpireFlowFiles method expireFlowFiles.

private void expireFlowFiles(final Connectable connectable) {
    // determine if the incoming connections for this Connectable have Expiration configured.
    boolean expirationConfigured = false;
    for (final Connection incomingConn : connectable.getIncomingConnections()) {
        if (FormatUtils.getTimeDuration(incomingConn.getFlowFileQueue().getFlowFileExpiration(), TimeUnit.MILLISECONDS) > 0) {
            expirationConfigured = true;
            break;
        }
    }
    // If expiration is not configured... don't bother running through the FlowFileQueue
    if (!expirationConfigured) {
        return;
    }
    final StandardProcessSession session = createSession(connectable);
    session.expireFlowFiles();
    session.commit();
}

Example 2 with StandardProcessSession

use of org.apache.nifi.controller.repository.StandardProcessSession in project nifi by apache.

the class ConnectableTask method invoke.

public InvocationResult invoke() {
    if (scheduleState.isTerminated()) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // make sure processor is not yielded
    if (isYielded()) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // make sure that either we're not clustered or this processor runs on all nodes or that this is the primary node
    if (!isRunOnCluster(flowController)) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // * All incoming connections are self-loops
    if (!isWorkToDo()) {
        return InvocationResult.yield("No work to do");
    }
    if (numRelationships > 0) {
        final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
        if (!repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships)) {
            return InvocationResult.yield("Backpressure Applied");
        }
    }
    final long batchNanos = connectable.getRunDuration(TimeUnit.NANOSECONDS);
    final ProcessSessionFactory sessionFactory;
    final StandardProcessSession rawSession;
    final boolean batch;
    if (connectable.isSessionBatchingSupported() && batchNanos > 0L) {
        rawSession = new StandardProcessSession(repositoryContext, scheduleState::isTerminated);
        sessionFactory = new BatchingSessionFactory(rawSession);
        batch = true;
    } else {
        rawSession = null;
        sessionFactory = new StandardProcessSessionFactory(repositoryContext, scheduleState::isTerminated);
        batch = false;
    }
    final ActiveProcessSessionFactory activeSessionFactory = new WeakHashMapProcessSessionFactory(sessionFactory);
    scheduleState.incrementActiveThreadCount(activeSessionFactory);
    final long startNanos = System.nanoTime();
    final long finishIfBackpressureEngaged = startNanos + (batchNanos / 25L);
    final long finishNanos = startNanos + batchNanos;
    int invocationCount = 0;
    final String originalThreadName = Thread.currentThread().getName();
    try {
        try (final AutoCloseable ncl = NarCloseable.withComponentNarLoader(connectable.getRunnableComponent().getClass(), connectable.getIdentifier())) {
            boolean shouldRun = connectable.getScheduledState() == ScheduledState.RUNNING;
            while (shouldRun) {
                connectable.onTrigger(processContext, activeSessionFactory);
                invocationCount++;
                if (!batch) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                final long nanoTime = System.nanoTime();
                if (nanoTime > finishNanos) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                if (nanoTime > finishIfBackpressureEngaged && isBackPressureEngaged()) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                if (connectable.getScheduledState() != ScheduledState.RUNNING) {
                    break;
                }
                if (!isWorkToDo()) {
                    break;
                }
                if (isYielded()) {
                    break;
                }
                if (numRelationships > 0) {
                    final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
                    shouldRun = repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships);
                }
            }
        } catch (final TerminatedTaskException tte) {
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.info("Failed to process session due to task being terminated", new Object[] { tte });
        } catch (final ProcessException pe) {
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.error("Failed to process session due to {}", new Object[] { pe });
        } catch (final Throwable t) {
            // Use ComponentLog to log the event so that a bulletin will be created for this processor
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.error("{} failed to process session due to {}; Processor Administratively Yielded for {}", new Object[] { connectable.getRunnableComponent(), t, schedulingAgent.getAdministrativeYieldDuration() }, t);
            logger.warn("Administratively Yielding {} due to uncaught Exception: {}", connectable.getRunnableComponent(), t.toString(), t);
            connectable.yield(schedulingAgent.getAdministrativeYieldDuration(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
        }
    } finally {
        try {
            if (batch) {
                try {
                    rawSession.commit();
                } catch (final Exception e) {
                    final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
                    procLog.error("Failed to commit session {} due to {}; rolling back", new Object[] { rawSession, e.toString() }, e);
                    try {
                        rawSession.rollback(true);
                    } catch (final Exception e1) {
                        procLog.error("Failed to roll back session {} due to {}", new Object[] { rawSession, e.toString() }, e);
                    }
                }
            }
            final long processingNanos = System.nanoTime() - startNanos;
            try {
                final StandardFlowFileEvent procEvent = new StandardFlowFileEvent(connectable.getIdentifier());
                procEvent.setProcessingNanos(processingNanos);
                procEvent.setInvocations(invocationCount);
                repositoryContext.getFlowFileEventRepository().updateRepository(procEvent);
            } catch (final IOException e) {
                logger.error("Unable to update FlowFileEvent Repository for {}; statistics may be inaccurate. Reason for failure: {}", connectable.getRunnableComponent(), e.toString());
                logger.error("", e);
            }
        } finally {
            scheduleState.decrementActiveThreadCount(activeSessionFactory);
            Thread.currentThread().setName(originalThreadName);
        }
    }
    return InvocationResult.DO_NOT_YIELD;
}