Examples with Processor org.apache.nifi.processor.Processor used on opensource projects

Example 36 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class StandardProcessorNode method getUndefinedRelationships.

public Set<Relationship> getUndefinedRelationships() {
    final Set<Relationship> undefined = new HashSet<>();
    final Set<Relationship> relationships;
    final Processor processor = processorRef.get().getProcessor();
    try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
        relationships = processor.getRelationships();
    }
    if (relationships == null) {
        return undefined;
    }
    for (final Relationship relation : relationships) {
        final Set<Connection> connectionSet = this.connections.get(relation);
        if (connectionSet == null || connectionSet.isEmpty()) {
            undefined.add(relation);
        }
    }
    return undefined;
}

Example 37 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class StandardProcessorNode method initiateStart.

private void initiateStart(final ScheduledExecutorService taskScheduler, final long administrativeYieldMillis, final ProcessContext processContext, final SchedulingAgentCallback schedulingAgentCallback) {
    final Processor processor = getProcessor();
    final ComponentLog procLog = new SimpleProcessLogger(StandardProcessorNode.this.getIdentifier(), processor);
    final long completionTimestamp = System.currentTimeMillis() + onScheduleTimeoutMillis;
    // Create a task to invoke the @OnScheduled annotation of the processor
    final Callable<Void> startupTask = () -> {
        LOG.debug("Invoking @OnScheduled methods of {}", processor);
        try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
            try {
                activateThread();
                try {
                    ReflectionUtils.invokeMethodsWithAnnotation(OnScheduled.class, processor, processContext);
                } finally {
                    deactivateThread();
                }
                if (scheduledState.compareAndSet(ScheduledState.STARTING, ScheduledState.RUNNING)) {
                    LOG.debug("Successfully completed the @OnScheduled methods of {}; will now start triggering processor to run", processor);
                    // callback provided by StandardProcessScheduler to essentially initiate component's onTrigger() cycle
                    schedulingAgentCallback.trigger();
                } else {
                    LOG.debug("Successfully invoked @OnScheduled methods of {} but scheduled state is no longer STARTING so will stop processor now", processor);
                    // can only happen if stopProcessor was called before service was transitioned to RUNNING state
                    activateThread();
                    try {
                        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnUnscheduled.class, processor, processContext);
                        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, processor, processContext);
                    } finally {
                        deactivateThread();
                    }
                    scheduledState.set(ScheduledState.STOPPED);
                }
            } finally {
                schedulingAgentCallback.onTaskComplete();
            }
        } catch (final Exception e) {
            procLog.error("Failed to properly initialize Processor. If still scheduled to run, NiFi will attempt to " + "initialize and run the Processor again after the 'Administrative Yield Duration' has elapsed. Failure is due to " + e, e);
            // If processor's task completed Exceptionally, then we want to retry initiating the start (if Processor is still scheduled to run).
            try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
                activateThread();
                try {
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnUnscheduled.class, processor, processContext);
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, processor, processContext);
                } finally {
                    deactivateThread();
                }
            }
            // make sure we only continue retry loop if STOP action wasn't initiated
            if (scheduledState.get() != ScheduledState.STOPPING) {
                // re-initiate the entire process
                final Runnable initiateStartTask = () -> initiateStart(taskScheduler, administrativeYieldMillis, processContext, schedulingAgentCallback);
                taskScheduler.schedule(initiateStartTask, administrativeYieldMillis, TimeUnit.MILLISECONDS);
            } else {
                scheduledState.set(ScheduledState.STOPPED);
            }
        }
        return null;
    };
    // Trigger the task in a background thread.
    final Future<?> taskFuture = schedulingAgentCallback.scheduleTask(startupTask);
    // Trigger a task periodically to check if @OnScheduled task completed. Once it has,
    // this task will call SchedulingAgentCallback#onTaskComplete.
    // However, if the task times out, we need to be able to cancel the monitoring. So, in order
    // to do this, we use #scheduleWithFixedDelay and then make that Future available to the task
    // itself by placing it into an AtomicReference.
    final AtomicReference<Future<?>> futureRef = new AtomicReference<>();
    final Runnable monitoringTask = new Runnable() {

        @Override
        public void run() {
            Future<?> monitoringFuture = futureRef.get();
            if (monitoringFuture == null) {
                // Future is not yet available. Just return and wait for the next invocation.
                return;
            }
            monitorAsyncTask(taskFuture, monitoringFuture, completionTimestamp);
        }
    };
    final Future<?> future = taskScheduler.scheduleWithFixedDelay(monitoringTask, 1, 10, TimeUnit.MILLISECONDS);
    futureRef.set(future);
}

Example 38 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class StandardProcessorNode method getRelationship.

/**
 * @param relationshipName
 *            name
 * @return the relationship for this nodes processor for the given name or
 *         creates a new relationship for the given name
 */
@Override
public Relationship getRelationship(final String relationshipName) {
    final Relationship specRel = new Relationship.Builder().name(relationshipName).build();
    Relationship returnRel = specRel;
    final Set<Relationship> relationships;
    final Processor processor = processorRef.get().getProcessor();
    try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
        relationships = processor.getRelationships();
    }
    for (final Relationship rel : relationships) {
        if (rel.equals(specRel)) {
            returnRel = rel;
            break;
        }
    }
    return returnRel;
}

Example 39 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class StandardProcessorNode method stop.

/**
 * Will idempotently stop the processor using the following sequence: <i>
 * <ul>
 * <li>Transition (atomically) Processor's scheduled state from RUNNING to
 * STOPPING. If the above state transition succeeds, then invoke any method
 * on the Processor with the {@link OnUnscheduled} annotation. Once those methods
 * have been called and returned (either normally or exceptionally), start checking
 * to see if all of the Processor's active threads have finished. If not, check again
 * every 100 milliseconds until they have.
 * Once all after threads have completed, the processor's @OnStopped operation will be invoked
 * and its scheduled state is set to STOPPED which completes processor stop
 * sequence.</li>
 * </ul>
 * </i>
 *
 * <p>
 * If for some reason processor's scheduled state can not be transitioned to
 * STOPPING (e.g., the processor didn't finish @OnScheduled operation when
 * stop was called), the attempt will be made to transition processor's
 * scheduled state from STARTING to STOPPING which will allow
 * {@link #start(ScheduledExecutorService, long, ProcessContext, Runnable)}
 * method to initiate processor's shutdown upon exiting @OnScheduled
 * operation, otherwise the processor's scheduled state will remain
 * unchanged ensuring that multiple calls to this method are idempotent.
 * </p>
 */
@Override
public CompletableFuture<Void> stop(final ProcessScheduler processScheduler, final ScheduledExecutorService executor, final ProcessContext processContext, final SchedulingAgent schedulingAgent, final LifecycleState scheduleState) {
    final Processor processor = processorRef.get().getProcessor();
    LOG.info("Stopping processor: " + processor.getClass());
    desiredState = ScheduledState.STOPPED;
    final CompletableFuture<Void> future = new CompletableFuture<>();
    if (this.scheduledState.compareAndSet(ScheduledState.RUNNING, ScheduledState.STOPPING)) {
        // will ensure that the Processor represented by this node can only be stopped once
        scheduleState.incrementActiveThreadCount(null);
        // will continue to monitor active threads, invoking OnStopped once there are no
        // active threads (with the exception of the thread performing shutdown operations)
        executor.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    if (scheduleState.isScheduled()) {
                        schedulingAgent.unschedule(StandardProcessorNode.this, scheduleState);
                        activateThread();
                        try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
                            ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnUnscheduled.class, processor, processContext);
                        } finally {
                            deactivateThread();
                        }
                    }
                    // all threads are complete if the active thread count is 1. This is because this thread that is
                    // performing the lifecycle actions counts as 1 thread.
                    final boolean allThreadsComplete = scheduleState.getActiveThreadCount() == 1;
                    if (allThreadsComplete) {
                        activateThread();
                        try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
                            ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, processor, processContext);
                        } finally {
                            deactivateThread();
                        }
                        scheduleState.decrementActiveThreadCount(null);
                        scheduledState.set(ScheduledState.STOPPED);
                        future.complete(null);
                        // Processor if need be.
                        if (desiredState == ScheduledState.RUNNING) {
                            processScheduler.startProcessor(StandardProcessorNode.this, true);
                        }
                    } else {
                        // Not all of the active threads have finished. Try again in 100 milliseconds.
                        executor.schedule(this, 100, TimeUnit.MILLISECONDS);
                    }
                } catch (final Exception e) {
                    LOG.warn("Failed while shutting down processor " + processor, e);
                }
            }
        });
    } else {
        // We do compareAndSet() instead of set() to ensure that Processor
        // stoppage is handled consistently including a condition where
        // Processor never got a chance to transition to RUNNING state
        // before stop() was called. If that happens the stop processor
        // routine will be initiated in start() method, otherwise the IF
        // part will handle the stop processor routine.
        this.scheduledState.compareAndSet(ScheduledState.STARTING, ScheduledState.STOPPING);
        future.complete(null);
    }
    return future;
}

Example 40 with Processor

use of org.apache.nifi.processor.Processor in project nifi by apache.

the class FlowController method reload.

@Override
public void reload(final ProcessorNode existingNode, final String newType, final BundleCoordinate bundleCoordinate, final Set<URL> additionalUrls) throws ProcessorInstantiationException {
    if (existingNode == null) {
        throw new IllegalStateException("Existing ProcessorNode cannot be null");
    }
    final String id = existingNode.getProcessor().getIdentifier();
    // ghost components will have a null logger
    if (existingNode.getLogger() != null) {
        existingNode.getLogger().debug("Reloading component {} to type {} from bundle {}", new Object[] { id, newType, bundleCoordinate });
    }
    // createProcessor will create a new instance class loader for the same id so
    // save the instance class loader to use it for calling OnRemoved on the existing processor
    final ClassLoader existingInstanceClassLoader = ExtensionManager.getInstanceClassLoader(id);
    // create a new node with firstTimeAdded as true so lifecycle methods get fired
    // attempt the creation to make sure it works before firing the OnRemoved methods below
    final ProcessorNode newNode = createProcessor(newType, id, bundleCoordinate, additionalUrls, true, false);
    // call OnRemoved for the existing processor using the previous instance class loader
    try (final NarCloseable x = NarCloseable.withComponentNarLoader(existingInstanceClassLoader)) {
        final StateManager stateManager = getStateManagerProvider().getStateManager(id);
        final StandardProcessContext processContext = new StandardProcessContext(existingNode, controllerServiceProvider, encryptor, stateManager, () -> false);
        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, existingNode.getProcessor(), processContext);
    } finally {
        ExtensionManager.closeURLClassLoader(id, existingInstanceClassLoader);
    }
    // set the new processor in the existing node
    final ComponentLog componentLogger = new SimpleProcessLogger(id, newNode.getProcessor());
    final TerminationAwareLogger terminationAwareLogger = new TerminationAwareLogger(componentLogger);
    LogRepositoryFactory.getRepository(id).setLogger(terminationAwareLogger);
    final LoggableComponent<Processor> newProcessor = new LoggableComponent<>(newNode.getProcessor(), newNode.getBundleCoordinate(), terminationAwareLogger);
    existingNode.setProcessor(newProcessor);
    existingNode.setExtensionMissing(newNode.isExtensionMissing());
    // need to refresh the properties in case we are changing from ghost component to real component
    existingNode.refreshProperties();
}