Examples with ComponentLog org.apache.nifi.logging.ComponentLog used on opensource projects

Example 86 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class HiveConnectionPool method onConfigured.

/**
 * Configures connection pool by creating an instance of the
 * {@link BasicDataSource} based on configuration provided with
 * {@link ConfigurationContext}.
 * <p>
 * This operation makes no guarantees that the actual connection could be
 * made since the underlying system may still go off-line during normal
 * operation of the connection pool.
 * <p/>
 * As of Apache NiFi 1.5.0, due to changes made to
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this class invoking
 * {@link HiveConfigurator#authenticate(Configuration, String, String)}
 * to authenticate a principal with Kerberos, Hive controller services no longer
 * attempt relogins explicitly.  For more information, please read the documentation for
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
 * <p/>
 * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started by
 * {@link HiveConfigurator#authenticate(Configuration, String, String, long)} when the Hive
 * controller service was enabled.  The use of a separate thread to explicitly relogin could cause race conditions
 * with the implicit relogin attempts made by hadoop/Hive code on a thread that references the same
 * {@link UserGroupInformation} instance.  One of these threads could leave the
 * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state
 * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed
 * authentication attempts that would leave the Hive controller service in an unrecoverable state.
 *
 * @see SecurityUtil#loginKerberos(Configuration, String, String)
 * @see HiveConfigurator#authenticate(Configuration, String, String)
 * @see HiveConfigurator#authenticate(Configuration, String, String, long)
 * @param context the configuration context
 * @throws InitializationException if unable to create a database connection
 */
@OnEnabled
public void onConfigured(final ConfigurationContext context) throws InitializationException {
    ComponentLog log = getLogger();
    final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).evaluateAttributeExpressions().getValue();
    final Configuration hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles);
    final String validationQuery = context.getProperty(VALIDATION_QUERY).evaluateAttributeExpressions().getValue();
    // add any dynamic properties to the Hive configuration
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            hiveConfig.set(descriptor.getName(), context.getProperty(descriptor).evaluateAttributeExpressions().getValue());
        }
    }
    final String drv = HiveDriver.class.getName();
    if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
        final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
        final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
        final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
        final String resolvedPrincipal;
        final String resolvedKeytab;
        if (credentialsService == null) {
            resolvedPrincipal = explicitPrincipal;
            resolvedKeytab = explicitKeytab;
        } else {
            resolvedPrincipal = credentialsService.getPrincipal();
            resolvedKeytab = credentialsService.getKeytab();
        }
        log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] { resolvedPrincipal, resolvedKeytab });
        try {
            ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab);
        } catch (AuthenticationFailedException ae) {
            log.error(ae.getMessage(), ae);
        }
        getLogger().info("Successfully logged in as principal {} with keytab {}", new Object[] { resolvedPrincipal, resolvedKeytab });
    }
    final String user = context.getProperty(DB_USER).evaluateAttributeExpressions().getValue();
    final String passw = context.getProperty(DB_PASSWORD).evaluateAttributeExpressions().getValue();
    final Long maxWaitMillis = context.getProperty(MAX_WAIT_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
    final Integer maxTotal = context.getProperty(MAX_TOTAL_CONNECTIONS).evaluateAttributeExpressions().asInteger();
    dataSource = new BasicDataSource();
    dataSource.setDriverClassName(drv);
    connectionUrl = context.getProperty(DATABASE_URL).evaluateAttributeExpressions().getValue();
    dataSource.setMaxWait(maxWaitMillis);
    dataSource.setMaxActive(maxTotal);
    if (validationQuery != null && !validationQuery.isEmpty()) {
        dataSource.setValidationQuery(validationQuery);
        dataSource.setTestOnBorrow(true);
    }
    dataSource.setUrl(connectionUrl);
    dataSource.setUsername(user);
    dataSource.setPassword(passw);
}

Example 87 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class StandardProcessorNode method start.

/**
 * Will idempotently start the processor using the following sequence: <i>
 * <ul>
 * <li>Validate Processor's state (e.g., PropertyDescriptors,
 * ControllerServices etc.)</li>
 * <li>Transition (atomically) Processor's scheduled state form STOPPED to
 * STARTING. If the above state transition succeeds, then execute the start
 * task (asynchronously) which will be re-tried until @OnScheduled is
 * executed successfully and "schedulingAgentCallback' is invoked, or until
 * STOP operation is initiated on this processor. If state transition fails
 * it means processor is already being started and WARN message will be
 * logged explaining it.</li>
 * </ul>
 * </i>
 * <p>
 * Any exception thrown while invoking operations annotated with @OnSchedule
 * will be caught and logged after which @OnUnscheduled operation will be
 * invoked (quietly) and the start sequence will be repeated (re-try) after
 * delay provided by 'administrativeYieldMillis'.
 * </p>
 * <p>
 * Upon successful completion of start sequence (@OnScheduled -&gt;
 * 'schedulingAgentCallback') the attempt will be made to transition
 * processor's scheduling state to RUNNING at which point processor is
 * considered to be fully started and functioning. If upon successful
 * invocation of @OnScheduled operation the processor can not be
 * transitioned to RUNNING state (e.g., STOP operation was invoked on the
 * processor while it's @OnScheduled operation was executing), the
 * processor's @OnUnscheduled operation will be invoked and its scheduling
 * state will be set to STOPPED at which point the processor is considered
 * to be fully stopped.
 * </p>
 */
@Override
public void start(final ScheduledExecutorService taskScheduler, final long administrativeYieldMillis, final ProcessContext processContext, final SchedulingAgentCallback schedulingAgentCallback, final boolean failIfStopping) {
    if (!this.isValid()) {
        throw new IllegalStateException("Processor " + this.getName() + " is not in a valid state due to " + this.getValidationErrors());
    }
    final Processor processor = processorRef.get().getProcessor();
    final ComponentLog procLog = new SimpleProcessLogger(StandardProcessorNode.this.getIdentifier(), processor);
    ScheduledState currentState;
    boolean starting;
    synchronized (this) {
        currentState = this.scheduledState.get();
        if (currentState == ScheduledState.STOPPED) {
            starting = this.scheduledState.compareAndSet(ScheduledState.STOPPED, ScheduledState.STARTING);
            if (starting) {
                desiredState = ScheduledState.RUNNING;
            }
        } else if (currentState == ScheduledState.STOPPING && !failIfStopping) {
            desiredState = ScheduledState.RUNNING;
            return;
        } else {
            starting = false;
        }
    }
    if (starting) {
        // will ensure that the Processor represented by this node can only be started once
        initiateStart(taskScheduler, administrativeYieldMillis, processContext, schedulingAgentCallback);
    } else {
        final String procName = processorRef.get().toString();
        LOG.warn("Cannot start {} because it is not currently stopped. Current state is {}", procName, currentState);
        procLog.warn("Cannot start {} because it is not currently stopped. Current state is {}", new Object[] { procName, currentState });
    }
}

Example 88 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class StandardProcessScheduler method unschedule.

@Override
public void unschedule(final ReportingTaskNode taskNode) {
    final LifecycleState lifecycleState = getLifecycleState(requireNonNull(taskNode), false);
    if (!lifecycleState.isScheduled()) {
        return;
    }
    taskNode.verifyCanStop();
    final SchedulingAgent agent = getSchedulingAgent(taskNode.getSchedulingStrategy());
    final ReportingTask reportingTask = taskNode.getReportingTask();
    taskNode.setScheduledState(ScheduledState.STOPPED);
    final Runnable unscheduleReportingTaskRunnable = new Runnable() {

        @Override
        public void run() {
            final ConfigurationContext configurationContext = taskNode.getConfigurationContext();
            synchronized (lifecycleState) {
                lifecycleState.setScheduled(false);
                try {
                    try (final NarCloseable x = NarCloseable.withComponentNarLoader(reportingTask.getClass(), reportingTask.getIdentifier())) {
                        ReflectionUtils.invokeMethodsWithAnnotation(OnUnscheduled.class, reportingTask, configurationContext);
                    }
                } catch (final Exception e) {
                    final Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e;
                    final ComponentLog componentLog = new SimpleProcessLogger(reportingTask.getIdentifier(), reportingTask);
                    componentLog.error("Failed to invoke @OnUnscheduled method due to {}", cause);
                    LOG.error("Failed to invoke the @OnUnscheduled methods of {} due to {}; administratively yielding this ReportingTask and will attempt to schedule it again after {}", reportingTask, cause.toString(), administrativeYieldDuration);
                    LOG.error("", cause);
                    try {
                        Thread.sleep(administrativeYieldMillis);
                    } catch (final InterruptedException ie) {
                    }
                }
                agent.unschedule(taskNode, lifecycleState);
                if (lifecycleState.getActiveThreadCount() == 0 && lifecycleState.mustCallOnStoppedMethods()) {
                    ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, reportingTask, configurationContext);
                }
            }
        }
    };
    componentLifeCycleThreadPool.execute(unscheduleReportingTaskRunnable);
}

Example 89 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class TestRollbackOnFailure method processInputs.

private void processInputs(RollbackOnFailure context, Integer[][] inputs, List<Integer> results) {
    final ExternalProcedure p = new ExternalProcedure();
    final MockComponentLog componentLog = new MockComponentLog("processor-id", this);
    final ExceptionHandler<RollbackOnFailure> handler = getContextAwareExceptionHandler(componentLog);
    for (Integer[] input : inputs) {
        if (!handler.execute(context, input, (in) -> {
            results.add(p.divide(in[0], in[1]));
            context.proceed();
        })) {
            continue;
        }
        assertEquals(input[2], results.get(results.size() - 1));
    }
}

Example 90 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class StandardControllerServiceProvider method createControllerService.

@Override
public ControllerServiceNode createControllerService(final String type, final String id, final BundleCoordinate bundleCoordinate, final Set<URL> additionalUrls, final boolean firstTimeAdded) {
    if (type == null || id == null || bundleCoordinate == null) {
        throw new NullPointerException();
    }
    ClassLoader cl = null;
    final ClassLoader currentContextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final Class<?> rawClass;
        try {
            final Bundle csBundle = ExtensionManager.getBundle(bundleCoordinate);
            if (csBundle == null) {
                throw new ControllerServiceInstantiationException("Unable to find bundle for coordinate " + bundleCoordinate.getCoordinate());
            }
            cl = ExtensionManager.createInstanceClassLoader(type, id, csBundle, additionalUrls);
            Thread.currentThread().setContextClassLoader(cl);
            rawClass = Class.forName(type, false, cl);
        } catch (final Exception e) {
            logger.error("Could not create Controller Service of type " + type + " for ID " + id + "; creating \"Ghost\" implementation", e);
            Thread.currentThread().setContextClassLoader(currentContextClassLoader);
            return createGhostControllerService(type, id, bundleCoordinate);
        }
        final Class<? extends ControllerService> controllerServiceClass = rawClass.asSubclass(ControllerService.class);
        final ControllerService originalService = controllerServiceClass.newInstance();
        final StandardControllerServiceInvocationHandler invocationHandler = new StandardControllerServiceInvocationHandler(originalService);
        // extract all interfaces... controllerServiceClass is non null so getAllInterfaces is non null
        final List<Class<?>> interfaceList = ClassUtils.getAllInterfaces(controllerServiceClass);
        final Class<?>[] interfaces = interfaceList.toArray(new Class<?>[interfaceList.size()]);
        final ControllerService proxiedService;
        if (cl == null) {
            proxiedService = (ControllerService) Proxy.newProxyInstance(getClass().getClassLoader(), interfaces, invocationHandler);
        } else {
            proxiedService = (ControllerService) Proxy.newProxyInstance(cl, interfaces, invocationHandler);
        }
        logger.info("Created Controller Service of type {} with identifier {}", type, id);
        final ComponentLog serviceLogger = new SimpleProcessLogger(id, originalService);
        final TerminationAwareLogger terminationAwareLogger = new TerminationAwareLogger(serviceLogger);
        originalService.initialize(new StandardControllerServiceInitializationContext(id, terminationAwareLogger, this, getStateManager(id), nifiProperties));
        final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(this, variableRegistry);
        final LoggableComponent<ControllerService> originalLoggableComponent = new LoggableComponent<>(originalService, bundleCoordinate, terminationAwareLogger);
        final LoggableComponent<ControllerService> proxiedLoggableComponent = new LoggableComponent<>(proxiedService, bundleCoordinate, terminationAwareLogger);
        final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry);
        final ControllerServiceNode serviceNode = new StandardControllerServiceNode(originalLoggableComponent, proxiedLoggableComponent, invocationHandler, id, validationContextFactory, this, componentVarRegistry, flowController);
        serviceNode.setName(rawClass.getSimpleName());
        invocationHandler.setServiceNode(serviceNode);
        if (firstTimeAdded) {
            try (final NarCloseable x = NarCloseable.withComponentNarLoader(originalService.getClass(), originalService.getIdentifier())) {
                ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, originalService);
            } catch (final Exception e) {
                throw new ComponentLifeCycleException("Failed to invoke On-Added Lifecycle methods of " + originalService, e);
            }
        }
        serviceCache.putIfAbsent(id, serviceNode);
        return serviceNode;
    } catch (final Throwable t) {
        throw new ControllerServiceInstantiationException(t);
    } finally {
        if (currentContextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(currentContextClassLoader);
        }
    }
}