Examples with SimpleProcessLogger org.apache.nifi.processor.SimpleProcessLogger used on opensource projects

Example 11 with SimpleProcessLogger

use of org.apache.nifi.processor.SimpleProcessLogger in project nifi by apache.

the class StandardFlowSynchronizer method getOrCreateReportingTask.

private ReportingTaskNode getOrCreateReportingTask(final FlowController controller, final ReportingTaskDTO dto, final boolean controllerInitialized, final boolean existingFlowEmpty) throws ReportingTaskInstantiationException {
    // create a new reporting task node when the controller is not initialized or the flow is empty
    if (!controllerInitialized || existingFlowEmpty) {
        BundleCoordinate coordinate;
        try {
            coordinate = BundleUtils.getCompatibleBundle(dto.getType(), dto.getBundle());
        } catch (final IllegalStateException e) {
            final BundleDTO bundleDTO = dto.getBundle();
            if (bundleDTO == null) {
                coordinate = BundleCoordinate.UNKNOWN_COORDINATE;
            } else {
                coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion());
            }
        }
        final ReportingTaskNode reportingTask = controller.createReportingTask(dto.getType(), dto.getId(), coordinate, false);
        reportingTask.setName(dto.getName());
        reportingTask.setComments(dto.getComments());
        reportingTask.setSchedulingPeriod(dto.getSchedulingPeriod());
        reportingTask.setSchedulingStrategy(SchedulingStrategy.valueOf(dto.getSchedulingStrategy()));
        reportingTask.setAnnotationData(dto.getAnnotationData());
        reportingTask.setProperties(dto.getProperties());
        final ComponentLog componentLog = new SimpleProcessLogger(dto.getId(), reportingTask.getReportingTask());
        final ReportingInitializationContext config = new StandardReportingInitializationContext(dto.getId(), dto.getName(), SchedulingStrategy.valueOf(dto.getSchedulingStrategy()), dto.getSchedulingPeriod(), componentLog, controller, nifiProperties, controller);
        try {
            reportingTask.getReportingTask().initialize(config);
        } catch (final InitializationException ie) {
            throw new ReportingTaskInstantiationException("Failed to initialize reporting task of type " + dto.getType(), ie);
        }
        return reportingTask;
    } else {
        // otherwise return the existing reporting task node
        return controller.getReportingTaskNode(dto.getId());
    }
}

Example 12 with SimpleProcessLogger

use of org.apache.nifi.processor.SimpleProcessLogger in project nifi by apache.

the class StandardProcessorNode method initiateStart.

private void initiateStart(final ScheduledExecutorService taskScheduler, final long administrativeYieldMillis, final ProcessContext processContext, final SchedulingAgentCallback schedulingAgentCallback) {
    final Processor processor = getProcessor();
    final ComponentLog procLog = new SimpleProcessLogger(StandardProcessorNode.this.getIdentifier(), processor);
    final long completionTimestamp = System.currentTimeMillis() + onScheduleTimeoutMillis;
    // Create a task to invoke the @OnScheduled annotation of the processor
    final Callable<Void> startupTask = () -> {
        LOG.debug("Invoking @OnScheduled methods of {}", processor);
        try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
            try {
                activateThread();
                try {
                    ReflectionUtils.invokeMethodsWithAnnotation(OnScheduled.class, processor, processContext);
                } finally {
                    deactivateThread();
                }
                if (scheduledState.compareAndSet(ScheduledState.STARTING, ScheduledState.RUNNING)) {
                    LOG.debug("Successfully completed the @OnScheduled methods of {}; will now start triggering processor to run", processor);
                    // callback provided by StandardProcessScheduler to essentially initiate component's onTrigger() cycle
                    schedulingAgentCallback.trigger();
                } else {
                    LOG.debug("Successfully invoked @OnScheduled methods of {} but scheduled state is no longer STARTING so will stop processor now", processor);
                    // can only happen if stopProcessor was called before service was transitioned to RUNNING state
                    activateThread();
                    try {
                        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnUnscheduled.class, processor, processContext);
                        ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, processor, processContext);
                    } finally {
                        deactivateThread();
                    }
                    scheduledState.set(ScheduledState.STOPPED);
                }
            } finally {
                schedulingAgentCallback.onTaskComplete();
            }
        } catch (final Exception e) {
            procLog.error("Failed to properly initialize Processor. If still scheduled to run, NiFi will attempt to " + "initialize and run the Processor again after the 'Administrative Yield Duration' has elapsed. Failure is due to " + e, e);
            // If processor's task completed Exceptionally, then we want to retry initiating the start (if Processor is still scheduled to run).
            try (final NarCloseable nc = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
                activateThread();
                try {
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnUnscheduled.class, processor, processContext);
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, processor, processContext);
                } finally {
                    deactivateThread();
                }
            }
            // make sure we only continue retry loop if STOP action wasn't initiated
            if (scheduledState.get() != ScheduledState.STOPPING) {
                // re-initiate the entire process
                final Runnable initiateStartTask = () -> initiateStart(taskScheduler, administrativeYieldMillis, processContext, schedulingAgentCallback);
                taskScheduler.schedule(initiateStartTask, administrativeYieldMillis, TimeUnit.MILLISECONDS);
            } else {
                scheduledState.set(ScheduledState.STOPPED);
            }
        }
        return null;
    };
    // Trigger the task in a background thread.
    final Future<?> taskFuture = schedulingAgentCallback.scheduleTask(startupTask);
    // Trigger a task periodically to check if @OnScheduled task completed. Once it has,
    // this task will call SchedulingAgentCallback#onTaskComplete.
    // However, if the task times out, we need to be able to cancel the monitoring. So, in order
    // to do this, we use #scheduleWithFixedDelay and then make that Future available to the task
    // itself by placing it into an AtomicReference.
    final AtomicReference<Future<?>> futureRef = new AtomicReference<>();
    final Runnable monitoringTask = new Runnable() {

        @Override
        public void run() {
            Future<?> monitoringFuture = futureRef.get();
            if (monitoringFuture == null) {
                // Future is not yet available. Just return and wait for the next invocation.
                return;
            }
            monitorAsyncTask(taskFuture, monitoringFuture, completionTimestamp);
        }
    };
    final Future<?> future = taskScheduler.scheduleWithFixedDelay(monitoringTask, 1, 10, TimeUnit.MILLISECONDS);
    futureRef.set(future);
}

Example 13 with SimpleProcessLogger

use of org.apache.nifi.processor.SimpleProcessLogger in project nifi by apache.

the class StandardControllerServiceNode method enable.

/**
 * Will atomically enable this service by invoking its @OnEnabled operation.
 * It uses CAS operation on {@link #stateRef} to transition this service
 * from DISABLED to ENABLING state. If such transition succeeds the service
 * will be marked as 'active' (see {@link ControllerServiceNode#isActive()}).
 * If such transition doesn't succeed then no enabling logic will be
 * performed and the method will exit. In other words it is safe to invoke
 * this operation multiple times and from multiple threads.
 * <br>
 * This operation will also perform re-try of service enabling in the event
 * of exception being thrown by previous invocation of @OnEnabled.
 * <br>
 * Upon successful invocation of @OnEnabled this service will be transitioned to
 * ENABLED state.
 * <br>
 * In the event where enabling took longer then expected by the user and such user
 * initiated disable operation, this service will be automatically disabled as soon
 * as it reached ENABLED state.
 */
@Override
public CompletableFuture<Void> enable(final ScheduledExecutorService scheduler, final long administrativeYieldMillis) {
    final CompletableFuture<Void> future = new CompletableFuture<>();
    if (this.stateTransition.transitionToEnabling(ControllerServiceState.DISABLED, future)) {
        synchronized (active) {
            this.active.set(true);
        }
        final StandardControllerServiceNode service = this;
        final ConfigurationContext configContext = new StandardConfigurationContext(this, this.serviceProvider, null, getVariableRegistry());
        scheduler.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    try (final NarCloseable nc = NarCloseable.withComponentNarLoader(getControllerServiceImplementation().getClass(), getIdentifier())) {
                        ReflectionUtils.invokeMethodsWithAnnotation(OnEnabled.class, getControllerServiceImplementation(), configContext);
                    }
                    boolean shouldEnable;
                    synchronized (active) {
                        shouldEnable = active.get() && stateTransition.enable();
                    }
                    if (!shouldEnable) {
                        LOG.debug("Disabling service {} after it has been enabled due to disable action being initiated.", service);
                        // Can only happen if user initiated DISABLE operation before service finished enabling. It's state will be
                        // set to DISABLING (see disable() operation)
                        invokeDisable(configContext);
                        stateTransition.disable();
                    } else {
                        LOG.debug("Successfully enabled {}", service);
                    }
                } catch (Exception e) {
                    future.completeExceptionally(e);
                    final Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e;
                    final ComponentLog componentLog = new SimpleProcessLogger(getIdentifier(), StandardControllerServiceNode.this);
                    componentLog.error("Failed to invoke @OnEnabled method due to {}", cause);
                    LOG.error("Failed to invoke @OnEnabled method of {} due to {}", getControllerServiceImplementation(), cause.toString());
                    invokeDisable(configContext);
                    if (isActive()) {
                        scheduler.schedule(this, administrativeYieldMillis, TimeUnit.MILLISECONDS);
                    } else {
                        try (final NarCloseable nc = NarCloseable.withComponentNarLoader(getControllerServiceImplementation().getClass(), getIdentifier())) {
                            ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnDisabled.class, getControllerServiceImplementation(), configContext);
                        }
                        stateTransition.disable();
                    }
                }
            }
        });
    } else {
        future.complete(null);
    }
    return future;
}

Example 14 with SimpleProcessLogger

use of org.apache.nifi.processor.SimpleProcessLogger in project nifi by apache.

the class StandardControllerServiceNode method invokeDisable.

/**
 */
private void invokeDisable(ConfigurationContext configContext) {
    try (final NarCloseable nc = NarCloseable.withComponentNarLoader(getControllerServiceImplementation().getClass(), getIdentifier())) {
        ReflectionUtils.invokeMethodsWithAnnotation(OnDisabled.class, StandardControllerServiceNode.this.getControllerServiceImplementation(), configContext);
        LOG.debug("Successfully disabled {}", this);
    } catch (Exception e) {
        final Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e;
        final ComponentLog componentLog = new SimpleProcessLogger(getIdentifier(), StandardControllerServiceNode.this);
        componentLog.error("Failed to invoke @OnDisabled method due to {}", cause);
        LOG.error("Failed to invoke @OnDisabled method of {} due to {}", getControllerServiceImplementation(), cause.toString());
    }
}

Example 15 with SimpleProcessLogger

use of org.apache.nifi.processor.SimpleProcessLogger in project nifi by apache.

the class ConnectableTask method invoke.

public InvocationResult invoke() {
    if (scheduleState.isTerminated()) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // make sure processor is not yielded
    if (isYielded()) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // make sure that either we're not clustered or this processor runs on all nodes or that this is the primary node
    if (!isRunOnCluster(flowController)) {
        return InvocationResult.DO_NOT_YIELD;
    }
    // * All incoming connections are self-loops
    if (!isWorkToDo()) {
        return InvocationResult.yield("No work to do");
    }
    if (numRelationships > 0) {
        final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
        if (!repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships)) {
            return InvocationResult.yield("Backpressure Applied");
        }
    }
    final long batchNanos = connectable.getRunDuration(TimeUnit.NANOSECONDS);
    final ProcessSessionFactory sessionFactory;
    final StandardProcessSession rawSession;
    final boolean batch;
    if (connectable.isSessionBatchingSupported() && batchNanos > 0L) {
        rawSession = new StandardProcessSession(repositoryContext, scheduleState::isTerminated);
        sessionFactory = new BatchingSessionFactory(rawSession);
        batch = true;
    } else {
        rawSession = null;
        sessionFactory = new StandardProcessSessionFactory(repositoryContext, scheduleState::isTerminated);
        batch = false;
    }
    final ActiveProcessSessionFactory activeSessionFactory = new WeakHashMapProcessSessionFactory(sessionFactory);
    scheduleState.incrementActiveThreadCount(activeSessionFactory);
    final long startNanos = System.nanoTime();
    final long finishIfBackpressureEngaged = startNanos + (batchNanos / 25L);
    final long finishNanos = startNanos + batchNanos;
    int invocationCount = 0;
    final String originalThreadName = Thread.currentThread().getName();
    try {
        try (final AutoCloseable ncl = NarCloseable.withComponentNarLoader(connectable.getRunnableComponent().getClass(), connectable.getIdentifier())) {
            boolean shouldRun = connectable.getScheduledState() == ScheduledState.RUNNING;
            while (shouldRun) {
                connectable.onTrigger(processContext, activeSessionFactory);
                invocationCount++;
                if (!batch) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                final long nanoTime = System.nanoTime();
                if (nanoTime > finishNanos) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                if (nanoTime > finishIfBackpressureEngaged && isBackPressureEngaged()) {
                    return InvocationResult.DO_NOT_YIELD;
                }
                if (connectable.getScheduledState() != ScheduledState.RUNNING) {
                    break;
                }
                if (!isWorkToDo()) {
                    break;
                }
                if (isYielded()) {
                    break;
                }
                if (numRelationships > 0) {
                    final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
                    shouldRun = repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships);
                }
            }
        } catch (final TerminatedTaskException tte) {
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.info("Failed to process session due to task being terminated", new Object[] { tte });
        } catch (final ProcessException pe) {
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.error("Failed to process session due to {}", new Object[] { pe });
        } catch (final Throwable t) {
            // Use ComponentLog to log the event so that a bulletin will be created for this processor
            final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
            procLog.error("{} failed to process session due to {}; Processor Administratively Yielded for {}", new Object[] { connectable.getRunnableComponent(), t, schedulingAgent.getAdministrativeYieldDuration() }, t);
            logger.warn("Administratively Yielding {} due to uncaught Exception: {}", connectable.getRunnableComponent(), t.toString(), t);
            connectable.yield(schedulingAgent.getAdministrativeYieldDuration(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
        }
    } finally {
        try {
            if (batch) {
                try {
                    rawSession.commit();
                } catch (final Exception e) {
                    final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
                    procLog.error("Failed to commit session {} due to {}; rolling back", new Object[] { rawSession, e.toString() }, e);
                    try {
                        rawSession.rollback(true);
                    } catch (final Exception e1) {
                        procLog.error("Failed to roll back session {} due to {}", new Object[] { rawSession, e.toString() }, e);
                    }
                }
            }
            final long processingNanos = System.nanoTime() - startNanos;
            try {
                final StandardFlowFileEvent procEvent = new StandardFlowFileEvent(connectable.getIdentifier());
                procEvent.setProcessingNanos(processingNanos);
                procEvent.setInvocations(invocationCount);
                repositoryContext.getFlowFileEventRepository().updateRepository(procEvent);
            } catch (final IOException e) {
                logger.error("Unable to update FlowFileEvent Repository for {}; statistics may be inaccurate. Reason for failure: {}", connectable.getRunnableComponent(), e.toString());
                logger.error("", e);
            }
        } finally {
            scheduleState.decrementActiveThreadCount(activeSessionFactory);
            Thread.currentThread().setName(originalThreadName);
        }
    }
    return InvocationResult.DO_NOT_YIELD;
}