Examples with Relationship org.apache.nifi.processor.Relationship used on opensource projects

Example 1 with Relationship

use of org.apache.nifi.processor.Relationship in project kylo by Teradata.

the class RegisterFeedTables method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    // Verify flow file exists
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    // Verify properties and attributes
    final String feedFormatOptions = Optional.ofNullable(context.getProperty(FEED_FORMAT_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).orElse(DEFAULT_FEED_FORMAT_OPTIONS);
    final String targetFormatOptions = Optional.ofNullable(context.getProperty(TARGET_FORMAT_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).orElse(DEFAULT_STORAGE_FORMAT);
    final String targetTableProperties = context.getProperty(TARGET_TBLPROPERTIES).evaluateAttributeExpressions(flowFile).getValue();
    final ColumnSpec[] partitions = Optional.ofNullable(context.getProperty(PARTITION_SPECS).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
    final String tableType = context.getProperty(TABLE_TYPE).getValue();
    final ColumnSpec[] columnSpecs = Optional.ofNullable(context.getProperty(FIELD_SPECIFICATION).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
    if (columnSpecs == null || columnSpecs.length == 0) {
        getLog().error("Missing field specification");
        session.transfer(flowFile, IngestProperties.REL_FAILURE);
        return;
    }
    ColumnSpec[] feedColumnSpecs = Optional.ofNullable(context.getProperty(FEED_FIELD_SPECIFICATION).evaluateAttributeExpressions(flowFile).getValue()).filter(StringUtils::isNotEmpty).map(ColumnSpec::createFromString).orElse(new ColumnSpec[0]);
    if (feedColumnSpecs == null || feedColumnSpecs.length == 0) {
        // Backwards compatibility with older templates we set the source and target to the same
        feedColumnSpecs = columnSpecs;
    }
    final String entity = context.getProperty(IngestProperties.FEED_NAME).evaluateAttributeExpressions(flowFile).getValue();
    if (entity == null || entity.isEmpty()) {
        getLog().error("Missing feed name");
        session.transfer(flowFile, IngestProperties.REL_FAILURE);
        return;
    }
    final String source = context.getProperty(IngestProperties.FEED_CATEGORY).evaluateAttributeExpressions(flowFile).getValue();
    if (source == null || source.isEmpty()) {
        getLog().error("Missing category name");
        session.transfer(flowFile, IngestProperties.REL_FAILURE);
        return;
    }
    final String feedRoot = context.getProperty(FEED_ROOT).evaluateAttributeExpressions(flowFile).getValue();
    final String profileRoot = context.getProperty(PROFILE_ROOT).evaluateAttributeExpressions(flowFile).getValue();
    final String masterRoot = context.getProperty(MASTER_ROOT).evaluateAttributeExpressions(flowFile).getValue();
    final TableRegisterConfiguration config = new TableRegisterConfiguration(feedRoot, profileRoot, masterRoot);
    // Register the tables
    final ThriftService thriftService = context.getProperty(THRIFT_SERVICE).asControllerService(ThriftService.class);
    try (final Connection conn = thriftService.getConnection()) {
        final TableRegisterSupport register = new TableRegisterSupport(conn, config);
        final boolean result;
        if (ALL_TABLES.equals(tableType)) {
            result = register.registerStandardTables(source, entity, feedColumnSpecs, feedFormatOptions, targetFormatOptions, partitions, columnSpecs, targetTableProperties);
        } else {
            result = register.registerTable(source, entity, feedColumnSpecs, feedFormatOptions, targetFormatOptions, partitions, columnSpecs, targetTableProperties, TableType.valueOf(tableType), true);
        }
        final Relationship relnResult = (result ? REL_SUCCESS : REL_FAILURE);
        session.transfer(flowFile, relnResult);
    } catch (final ProcessException | SQLException e) {
        getLog().error("Unable to obtain connection for {} due to {}; routing to failure", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 2 with Relationship

use of org.apache.nifi.processor.Relationship in project kylo by Teradata.

the class ExecutePySpark method init.

@Override
protected void init(@Nonnull final ProcessorInitializationContext context) {
    super.init(context);
    /* Create Kerberos properties */
    final SpringSecurityContextLoader securityContextLoader = SpringSecurityContextLoader.create(context);
    final KerberosProperties kerberosProperties = securityContextLoader.getKerberosProperties();
    KERBEROS_KEYTAB = kerberosProperties.createKerberosKeytabProperty();
    KERBEROS_PRINCIPAL = kerberosProperties.createKerberosPrincipalProperty();
    /* Create list of properties */
    final List<PropertyDescriptor> properties = new ArrayList<>();
    properties.add(KERBEROS_PRINCIPAL);
    properties.add(KERBEROS_KEYTAB);
    properties.add(HADOOP_CONFIGURATION_RESOURCES);
    properties.add(PYSPARK_APP_FILE);
    properties.add(PYSPARK_APP_ARGS);
    properties.add(PYSPARK_APP_NAME);
    properties.add(PYSPARK_ADDITIONAL_FILES);
    properties.add(SPARK_MASTER);
    properties.add(SPARK_YARN_DEPLOY_MODE);
    properties.add(YARN_QUEUE);
    properties.add(SPARK_HOME);
    properties.add(DRIVER_MEMORY);
    properties.add(EXECUTOR_MEMORY);
    properties.add(EXECUTOR_INSTANCES);
    properties.add(EXECUTOR_CORES);
    properties.add(NETWORK_TIMEOUT);
    properties.add(ADDITIONAL_SPARK_CONFIG_OPTIONS);
    this.properties = Collections.unmodifiableList(properties);
    /* Create list of relationships */
    final Set<Relationship> relationships = new HashSet<>();
    relationships.add(REL_SUCCESS);
    relationships.add(REL_FAILURE);
    this.relationships = Collections.unmodifiableSet(relationships);
}

Example 3 with Relationship

use of org.apache.nifi.processor.Relationship in project kylo by Teradata.

the class ExportSqoop method init.

@Override
protected void init(@Nonnull final ProcessorInitializationContext context) {
    super.init(context);
    /* Create Kerberos properties */
    final SpringSecurityContextLoader securityContextLoader = SpringSecurityContextLoader.create(context);
    final KerberosProperties kerberosProperties = securityContextLoader.getKerberosProperties();
    KERBEROS_KEYTAB = kerberosProperties.createKerberosKeytabProperty();
    KERBEROS_PRINCIPAL = kerberosProperties.createKerberosPrincipalProperty();
    /* Create list of properties */
    final List<PropertyDescriptor> properties = new ArrayList<>();
    properties.add(KERBEROS_PRINCIPAL);
    properties.add(KERBEROS_KEYTAB);
    properties.add(SQOOP_CONNECTION_SERVICE);
    properties.add(SOURCE_HDFS_DIRECTORY);
    properties.add(SOURCE_HDFS_FILE_DELIMITER);
    properties.add(SOURCE_NULL_INTERPRETATION_STRATEGY);
    properties.add(SOURCE_NULL_CUSTOM_STRING_IDENTIFIER);
    properties.add(SOURCE_NULL_CUSTOM_NON_STRING_IDENTIFIER);
    properties.add(HCATALOG_DATABASE);
    properties.add(HCATALOG_TABLE);
    properties.add(SQOOP_SYSTEM_PROPERTIES);
    properties.add(SQOOP_ADDITIONAL_ARGUMENTS);
    properties.add(TARGET_TABLE_NAME);
    properties.add(CLUSTER_MAP_TASKS);
    this.properties = Collections.unmodifiableList(properties);
    /* Create list of relationships */
    final Set<Relationship> relationships = new HashSet<>();
    relationships.add(REL_SUCCESS);
    relationships.add(REL_FAILURE);
    this.relationships = Collections.unmodifiableSet(relationships);
}

Example 4 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class DtoFactory method createProcessorDto.

/**
 * Creates a ProcessorDTO from the specified ProcessorNode.
 *
 * @param node node
 * @return dto
 */
public ProcessorDTO createProcessorDto(final ProcessorNode node) {
    if (node == null) {
        return null;
    }
    final BundleCoordinate bundleCoordinate = node.getBundleCoordinate();
    final List<Bundle> compatibleBundles = ExtensionManager.getBundles(node.getCanonicalClassName()).stream().filter(bundle -> {
        final BundleCoordinate coordinate = bundle.getBundleDetails().getCoordinate();
        return bundleCoordinate.getGroup().equals(coordinate.getGroup()) && bundleCoordinate.getId().equals(coordinate.getId());
    }).collect(Collectors.toList());
    final ProcessorDTO dto = new ProcessorDTO();
    dto.setId(node.getIdentifier());
    dto.setPosition(createPositionDto(node.getPosition()));
    dto.setStyle(node.getStyle());
    dto.setParentGroupId(node.getProcessGroup().getIdentifier());
    dto.setInputRequirement(node.getInputRequirement().name());
    dto.setPersistsState(node.getProcessor().getClass().isAnnotationPresent(Stateful.class));
    dto.setRestricted(node.isRestricted());
    dto.setDeprecated(node.isDeprecated());
    dto.setExtensionMissing(node.isExtensionMissing());
    dto.setMultipleVersionsAvailable(compatibleBundles.size() > 1);
    dto.setVersionedComponentId(node.getVersionedComponentId().orElse(null));
    dto.setType(node.getCanonicalClassName());
    dto.setBundle(createBundleDto(bundleCoordinate));
    dto.setName(node.getName());
    dto.setState(node.getScheduledState().toString());
    // build the relationship dtos
    final List<RelationshipDTO> relationships = new ArrayList<>();
    for (final Relationship rel : node.getRelationships()) {
        final RelationshipDTO relationshipDTO = new RelationshipDTO();
        relationshipDTO.setDescription(rel.getDescription());
        relationshipDTO.setName(rel.getName());
        relationshipDTO.setAutoTerminate(node.isAutoTerminated(rel));
        relationships.add(relationshipDTO);
    }
    // sort the relationships
    Collections.sort(relationships, new Comparator<RelationshipDTO>() {

        @Override
        public int compare(final RelationshipDTO r1, final RelationshipDTO r2) {
            return Collator.getInstance(Locale.US).compare(r1.getName(), r2.getName());
        }
    });
    // set the relationships
    dto.setRelationships(relationships);
    dto.setDescription(getCapabilityDescription(node.getClass()));
    dto.setSupportsParallelProcessing(!node.isTriggeredSerially());
    dto.setSupportsEventDriven(node.isEventDrivenSupported());
    dto.setSupportsBatching(node.isSessionBatchingSupported());
    dto.setConfig(createProcessorConfigDto(node));
    final Collection<ValidationResult> validationErrors = node.getValidationErrors();
    if (validationErrors != null && !validationErrors.isEmpty()) {
        final List<String> errors = new ArrayList<>();
        for (final ValidationResult validationResult : validationErrors) {
            errors.add(validationResult.toString());
        }
        dto.setValidationErrors(errors);
    }
    return dto;
}

Example 5 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class ControllerSearchService method search.

private ComponentSearchResultDTO search(final String searchStr, final ProcessorNode procNode) {
    final List<String> matches = new ArrayList<>();
    final Processor processor = procNode.getProcessor();
    addIfAppropriate(searchStr, procNode.getIdentifier(), "Id", matches);
    addIfAppropriate(searchStr, procNode.getVersionedComponentId().orElse(null), "Version Control ID", matches);
    addIfAppropriate(searchStr, procNode.getName(), "Name", matches);
    addIfAppropriate(searchStr, procNode.getComments(), "Comments", matches);
    // consider scheduling strategy
    if (SchedulingStrategy.EVENT_DRIVEN.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("event", searchStr)) {
        matches.add("Scheduling strategy: Event driven");
    } else if (SchedulingStrategy.TIMER_DRIVEN.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("timer", searchStr)) {
        matches.add("Scheduling strategy: Timer driven");
    } else if (SchedulingStrategy.PRIMARY_NODE_ONLY.equals(procNode.getSchedulingStrategy()) && StringUtils.containsIgnoreCase("primary", searchStr)) {
        // PRIMARY_NODE_ONLY has been deprecated as a SchedulingStrategy and replaced by PRIMARY as an ExecutionNode.
        matches.add("Scheduling strategy: On primary node");
    }
    // consider execution node
    if (ExecutionNode.PRIMARY.equals(procNode.getExecutionNode()) && StringUtils.containsIgnoreCase("primary", searchStr)) {
        matches.add("Execution node: primary");
    }
    // consider scheduled state
    if (ScheduledState.DISABLED.equals(procNode.getScheduledState())) {
        if (StringUtils.containsIgnoreCase("disabled", searchStr)) {
            matches.add("Run status: Disabled");
        }
    } else {
        if (StringUtils.containsIgnoreCase("invalid", searchStr) && !procNode.isValid()) {
            matches.add("Run status: Invalid");
        } else if (ScheduledState.RUNNING.equals(procNode.getScheduledState()) && StringUtils.containsIgnoreCase("running", searchStr)) {
            matches.add("Run status: Running");
        } else if (ScheduledState.STOPPED.equals(procNode.getScheduledState()) && StringUtils.containsIgnoreCase("stopped", searchStr)) {
            matches.add("Run status: Stopped");
        }
    }
    for (final Relationship relationship : procNode.getRelationships()) {
        addIfAppropriate(searchStr, relationship.getName(), "Relationship", matches);
    }
    // Add both the actual class name and the component type. This allows us to search for 'Ghost'
    // to search for components that could not be instantiated.
    addIfAppropriate(searchStr, processor.getClass().getSimpleName(), "Type", matches);
    addIfAppropriate(searchStr, procNode.getComponentType(), "Type", matches);
    for (final Map.Entry<PropertyDescriptor, String> entry : procNode.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = entry.getKey();
        addIfAppropriate(searchStr, descriptor.getName(), "Property name", matches);
        addIfAppropriate(searchStr, descriptor.getDescription(), "Property description", matches);
        // never include sensitive properties values in search results
        if (descriptor.isSensitive()) {
            continue;
        }
        String value = entry.getValue();
        // if unset consider default value
        if (value == null) {
            value = descriptor.getDefaultValue();
        }
        // evaluate if the value matches the search criteria
        if (StringUtils.containsIgnoreCase(value, searchStr)) {
            matches.add("Property value: " + descriptor.getName() + " - " + value);
        }
    }
    // consider searching the processor directly
    if (processor instanceof Searchable) {
        final Searchable searchable = (Searchable) processor;
        final SearchContext context = new StandardSearchContext(searchStr, procNode, flowController, variableRegistry);
        // search the processor using the appropriate thread context classloader
        try (final NarCloseable x = NarCloseable.withComponentNarLoader(processor.getClass(), processor.getIdentifier())) {
            final Collection<SearchResult> searchResults = searchable.search(context);
            if (CollectionUtils.isNotEmpty(searchResults)) {
                for (final SearchResult searchResult : searchResults) {
                    matches.add(searchResult.getLabel() + ": " + searchResult.getMatch());
                }
            }
        } catch (final Throwable t) {
        // log this as error
        }
    }
    if (matches.isEmpty()) {
        return null;
    }
    final ComponentSearchResultDTO result = new ComponentSearchResultDTO();
    result.setId(procNode.getIdentifier());
    result.setMatches(matches);
    result.setName(procNode.getName());
    return result;
}