Examples with PropertyDescriptor org.apache.nifi.components.PropertyDescriptor used on opensource projects

Example 81 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class ExecuteGroovyScript method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession _session) throws ProcessException {
    boolean toFailureOnError = VALID_FAIL_STRATEGY[1].equals(context.getProperty(FAIL_STRATEGY).getValue());
    // create wrapped session to control list of newly created and files got from this session.
    // so transfer original input to failure will be possible
    GroovyProcessSessionWrap session = new GroovyProcessSessionWrap(_session, toFailureOnError);
    HashMap CTL = new AccessMap("CTL");
    HashMap SQL = new AccessMap("SQL");
    try {
        // compilation must be moved to validation
        Script script = getGroovyScript();
        Map bindings = script.getBinding().getVariables();
        bindings.clear();
        // Find the user-added properties and bind them for the script
        for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
            if (property.getKey().isDynamic()) {
                if (property.getKey().getName().startsWith("CTL.")) {
                    // get controller service
                    ControllerService ctl = context.getProperty(property.getKey()).asControllerService(ControllerService.class);
                    CTL.put(property.getKey().getName().substring(4), ctl);
                } else if (property.getKey().getName().startsWith("SQL.")) {
                    DBCPService dbcp = context.getProperty(property.getKey()).asControllerService(DBCPService.class);
                    SQL.put(property.getKey().getName().substring(4), dbcp);
                } else {
                    // Add the dynamic property bound to its full PropertyValue to the script engine
                    if (property.getValue() != null) {
                        bindings.put(property.getKey().getName(), context.getProperty(property.getKey()));
                    }
                }
            }
        }
        onInitSQL(SQL);
        bindings.put("session", session);
        bindings.put("context", context);
        bindings.put("log", getLogger());
        bindings.put("REL_SUCCESS", REL_SUCCESS);
        bindings.put("REL_FAILURE", REL_FAILURE);
        bindings.put("CTL", CTL);
        bindings.put("SQL", SQL);
        script.run();
        bindings.clear();
        onCommitSQL(SQL);
        session.commit();
    } catch (Throwable t) {
        getLogger().error(t.toString(), t);
        onFailSQL(SQL);
        if (toFailureOnError) {
            // transfer all received to failure with two new attributes: ERROR_MESSAGE and ERROR_STACKTRACE.
            session.revertReceivedTo(REL_FAILURE, StackTraceUtils.deepSanitize(t));
        } else {
            session.rollback(true);
        }
    } finally {
        onFinitSQL(SQL);
    }
}

Example 82 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class StandardSnippetDAO method lookupSensitiveProcessorProperties.

private void lookupSensitiveProcessorProperties(final Set<ProcessorDTO> processors) {
    final ProcessGroup rootGroup = flowController.getGroup(flowController.getRootGroupId());
    // go through each processor
    for (final ProcessorDTO processorDTO : processors) {
        final ProcessorConfigDTO processorConfig = processorDTO.getConfig();
        // ensure that some property configuration have been specified
        if (processorConfig != null && processorConfig.getProperties() != null) {
            final Map<String, String> processorProperties = processorConfig.getProperties();
            // find the corresponding processor
            final ProcessorNode processorNode = rootGroup.findProcessor(processorDTO.getId());
            if (processorNode == null) {
                throw new IllegalArgumentException(String.format("Unable to create snippet because Processor '%s' could not be found", processorDTO.getId()));
            }
            // look for sensitive properties get the actual value
            for (Entry<PropertyDescriptor, String> entry : processorNode.getProperties().entrySet()) {
                final PropertyDescriptor descriptor = entry.getKey();
                if (descriptor.isSensitive()) {
                    processorProperties.put(descriptor.getName(), entry.getValue());
                }
            }
        }
    }
}

Example 83 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class JMSConnectionFactoryProvider method setConnectionFactoryProperties.

/**
 * This operation follows standard bean convention by matching property name
 * to its corresponding 'setter' method. Once the method was located it is
 * invoked to set the corresponding property to a value provided by during
 * service configuration. For example, 'channel' property will correspond to
 * 'setChannel(..) method and 'queueManager' property will correspond to
 * setQueueManager(..) method with a single argument.
 * <p>
 * There are also few adjustments to accommodate well known brokers. For
 * example ActiveMQ ConnectionFactory accepts address of the Message Broker
 * in a form of URL while IBMs in the form of host/port pair (more common).
 * So this method will use value retrieved from the 'BROKER_URI' static
 * property 'as is' if ConnectionFactory implementation is coming from
 * ActiveMQ and for all others (for now) the 'BROKER_URI' value will be
 * split on ':' and the resulting pair will be used to execute
 * setHostName(..) and setPort(..) methods on the provided
 * ConnectionFactory. This may need to be maintained and adjusted to
 * accommodate other implementation of ConnectionFactory, but only for
 * URL/Host/Port issue. All other properties are set as dynamic properties
 * where user essentially provides both property name and value, The bean
 * convention is also explained in user manual for this component with links
 * pointing to documentation of various ConnectionFactories.
 *
 * @see #setProperty(String, String) method
 */
private void setConnectionFactoryProperties(ConfigurationContext context) {
    for (final Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        PropertyDescriptor descriptor = entry.getKey();
        String propertyName = descriptor.getName();
        if (descriptor.isDynamic()) {
            this.setProperty(propertyName, entry.getValue());
        } else {
            if (propertyName.equals(BROKER)) {
                String brokerValue = context.getProperty(descriptor).evaluateAttributeExpressions().getValue();
                if (context.getProperty(CONNECTION_FACTORY_IMPL).evaluateAttributeExpressions().getValue().startsWith("org.apache.activemq")) {
                    this.setProperty("brokerURL", brokerValue);
                } else {
                    String[] hostPort = brokerValue.split(":");
                    if (hostPort.length == 2) {
                        this.setProperty("hostName", hostPort[0]);
                        this.setProperty("port", hostPort[1]);
                    } else if (hostPort.length != 2) {
                        // for tibco
                        this.setProperty("serverUrl", brokerValue);
                    } else {
                        throw new IllegalArgumentException("Failed to parse broker url: " + brokerValue);
                    }
                }
                SSLContextService sc = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
                if (sc != null) {
                    SSLContext ssl = sc.createSSLContext(ClientAuth.NONE);
                    this.setProperty("sSLSocketFactory", ssl.getSocketFactory());
                }
            }
        // ignore 'else', since it's the only non-dynamic property that is relevant to CF configuration
        }
    }
}

Example 84 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class GetKafka method createConsumers.

public void createConsumers(final ProcessContext context) {
    final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions().getValue();
    final Properties props = new Properties();
    props.setProperty("zookeeper.connect", context.getProperty(ZOOKEEPER_CONNECTION_STRING).evaluateAttributeExpressions().getValue());
    props.setProperty("group.id", context.getProperty(GROUP_ID).evaluateAttributeExpressions().getValue());
    props.setProperty("client.id", context.getProperty(CLIENT_NAME).getValue());
    props.setProperty("auto.commit.interval.ms", String.valueOf(context.getProperty(ZOOKEEPER_COMMIT_DELAY).asTimePeriod(TimeUnit.MILLISECONDS)));
    props.setProperty("auto.offset.reset", context.getProperty(AUTO_OFFSET_RESET).getValue());
    props.setProperty("zookeeper.connection.timeout.ms", context.getProperty(ZOOKEEPER_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).toString());
    props.setProperty("socket.timeout.ms", context.getProperty(KAFKA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).toString());
    for (final Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        PropertyDescriptor descriptor = entry.getKey();
        if (descriptor.isDynamic()) {
            if (props.containsKey(descriptor.getName())) {
                this.getLogger().warn("Overriding existing property '" + descriptor.getName() + "' which had value of '" + props.getProperty(descriptor.getName()) + "' with dynamically set value '" + entry.getValue() + "'.");
            }
            props.setProperty(descriptor.getName(), entry.getValue());
        }
    }
    /*
         * Unless user sets it to some explicit value we are setting it to the
         * lowest possible value of 1 millisecond to ensure the
         * consumerStream.hasNext() doesn't block. See
         * http://kafka.apache.org/documentation.html#configuration) as well as
         * comment in 'catch ConsumerTimeoutException' in onTrigger() for more
         * explanation as to the reasoning behind it.
         */
    if (!props.containsKey("consumer.timeout.ms")) {
        this.getLogger().info("Setting 'consumer.timeout.ms' to 1 milliseconds to avoid consumer" + " block in the event when no events are present in Kafka topic. If you wish to change this value " + " set it as dynamic property. If you wish to explicitly enable consumer block (at your own risk)" + " set its value to -1.");
        props.setProperty("consumer.timeout.ms", "1");
    }
    int partitionCount = KafkaUtils.retrievePartitionCountForTopic(context.getProperty(ZOOKEEPER_CONNECTION_STRING).evaluateAttributeExpressions().getValue(), context.getProperty(TOPIC).evaluateAttributeExpressions().getValue());
    final ConsumerConfig consumerConfig = new ConsumerConfig(props);
    consumer = Consumer.createJavaConsumerConnector(consumerConfig);
    final Map<String, Integer> topicCountMap = new HashMap<>(1);
    int concurrentTaskToUse = context.getMaxConcurrentTasks();
    if (context.getMaxConcurrentTasks() < partitionCount) {
        this.getLogger().warn("The amount of concurrent tasks '" + context.getMaxConcurrentTasks() + "' configured for " + "this processor is less than the amount of partitions '" + partitionCount + "' for topic '" + context.getProperty(TOPIC).evaluateAttributeExpressions().getValue() + "'. " + "Consider making it equal to the amount of partition count for most efficient event consumption.");
    } else if (context.getMaxConcurrentTasks() > partitionCount) {
        concurrentTaskToUse = partitionCount;
        this.getLogger().warn("The amount of concurrent tasks '" + context.getMaxConcurrentTasks() + "' configured for " + "this processor is greater than the amount of partitions '" + partitionCount + "' for topic '" + context.getProperty(TOPIC).evaluateAttributeExpressions().getValue() + "'. " + "Therefore those tasks would never see a message. To avoid that the '" + partitionCount + "'(partition count) will be used to consume events");
    }
    topicCountMap.put(topic, concurrentTaskToUse);
    final Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
    final List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);
    this.streamIterators.clear();
    for (final KafkaStream<byte[], byte[]> stream : streams) {
        streamIterators.add(stream.iterator());
    }
    this.consumerStreamsReady.set(true);
}

Example 85 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class StandardProcessGroup method updateControllerService.

private void updateControllerService(final ControllerServiceNode service, final VersionedControllerService proposed) {
    service.setAnnotationData(proposed.getAnnotationData());
    service.setComments(proposed.getComments());
    service.setName(proposed.getName());
    final Map<String, String> properties = populatePropertiesMap(service.getProperties(), proposed.getProperties(), proposed.getPropertyDescriptors(), service.getProcessGroup());
    service.setProperties(properties, true);
    if (!isEqual(service.getBundleCoordinate(), proposed.getBundle())) {
        final BundleCoordinate newBundleCoordinate = toCoordinate(proposed.getBundle());
        final List<PropertyDescriptor> descriptors = new ArrayList<>(service.getProperties().keySet());
        final Set<URL> additionalUrls = service.getAdditionalClasspathResources(descriptors);
        flowController.reload(service, proposed.getType(), newBundleCoordinate, additionalUrls);
    }
}