Examples with Relationship org.apache.nifi.processor.Relationship used on opensource projects

Example 56 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 57 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class FlowController method createConnection.

/**
 * Creates a connection between two Connectable objects.
 *
 * @param id required ID of the connection
 * @param name the name of the connection, or <code>null</code> to leave the
 * connection unnamed
 * @param source required source
 * @param destination required destination
 * @param relationshipNames required collection of relationship names
 * @return
 *
 * @throws NullPointerException if the ID, source, destination, or set of
 * relationships is null.
 * @throws IllegalArgumentException if <code>relationships</code> is an
 * empty collection
 */
public Connection createConnection(final String id, final String name, final Connectable source, final Connectable destination, final Collection<String> relationshipNames) {
    final StandardConnection.Builder builder = new StandardConnection.Builder(processScheduler);
    final List<Relationship> relationships = new ArrayList<>();
    for (final String relationshipName : requireNonNull(relationshipNames)) {
        relationships.add(new Relationship.Builder().name(relationshipName).build());
    }
    // Create and initialize a FlowFileSwapManager for this connection
    final FlowFileSwapManager swapManager = createSwapManager(nifiProperties);
    final EventReporter eventReporter = createEventReporter(getBulletinRepository());
    try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
        final SwapManagerInitializationContext initializationContext = new SwapManagerInitializationContext() {

            @Override
            public ResourceClaimManager getResourceClaimManager() {
                return resourceClaimManager;
            }

            @Override
            public FlowFileRepository getFlowFileRepository() {
                return flowFileRepository;
            }

            @Override
            public EventReporter getEventReporter() {
                return eventReporter;
            }
        };
        swapManager.initialize(initializationContext);
    }
    return builder.id(requireNonNull(id).intern()).name(name == null ? null : name.intern()).relationships(relationships).source(requireNonNull(source)).destination(destination).swapManager(swapManager).queueSwapThreshold(nifiProperties.getQueueSwapThreshold()).eventReporter(eventReporter).resourceClaimManager(resourceClaimManager).flowFileRepository(flowFileRepository).provenanceRepository(provenanceRepository).build();
}

Example 58 with Relationship

use of org.apache.nifi.processor.Relationship in project kylo by Teradata.

the class ImportSqoop method init.

@Override
protected void init(@Nonnull final ProcessorInitializationContext context) {
    super.init(context);
    /* Create Kerberos properties */
    final SpringSecurityContextLoader securityContextLoader = SpringSecurityContextLoader.create(context);
    final KerberosProperties kerberosProperties = securityContextLoader.getKerberosProperties();
    KERBEROS_KEYTAB = kerberosProperties.createKerberosKeytabProperty();
    KERBEROS_PRINCIPAL = kerberosProperties.createKerberosPrincipalProperty();
    /* Create list of properties */
    final List<PropertyDescriptor> properties = new ArrayList<>();
    properties.add(KERBEROS_PRINCIPAL);
    properties.add(KERBEROS_KEYTAB);
    properties.add(SQOOP_CONNECTION_SERVICE);
    properties.add(SOURCE_TABLE_NAME);
    properties.add(SOURCE_TABLE_FIELDS);
    properties.add(SOURCE_TABLE_WHERE_CLAUSE);
    properties.add(SOURCE_LOAD_STRATEGY);
    properties.add(SOURCE_CHECK_COLUMN_NAME);
    properties.add(SOURCE_CHECK_COLUMN_LAST_VALUE);
    properties.add(SOURCE_PROPERTY_WATERMARK);
    properties.add(SOURCE_SPLIT_BY_FIELD);
    properties.add(SOURCE_BOUNDARY_QUERY);
    properties.add(CLUSTER_MAP_TASKS);
    properties.add(CLUSTER_UI_JOB_NAME);
    properties.add(TARGET_HDFS_DIRECTORY);
    properties.add(TARGET_HDFS_DIRECTORY_EXISTS_STRATEGY);
    properties.add(TARGET_EXTRACT_DATA_FORMAT);
    properties.add(TARGET_HDFS_FILE_FIELD_DELIMITER);
    properties.add(TARGET_HDFS_FILE_RECORD_DELIMITER);
    properties.add(TARGET_HIVE_DELIM_STRATEGY);
    properties.add(TARGET_HIVE_REPLACE_DELIM);
    properties.add(TARGET_COMPRESSION_ALGORITHM);
    properties.add(TARGET_COLUMN_TYPE_MAPPING);
    properties.add(SQOOP_CODEGEN_DIR);
    properties.add(SOURCESPECIFIC_SQLSERVER_SCHEMA);
    properties.add(SQOOP_SYSTEM_PROPERTIES);
    properties.add(SQOOP_ADDITIONAL_ARGUMENTS);
    this.properties = Collections.unmodifiableList(properties);
    /* Create list of relationships */
    final Set<Relationship> relationships = new HashSet<>();
    relationships.add(REL_SUCCESS);
    relationships.add(REL_FAILURE);
    this.relationships = Collections.unmodifiableSet(relationships);
}

Example 59 with Relationship

use of org.apache.nifi.processor.Relationship in project kylo by Teradata.

the class InitializeFeed method beginInitialization.

private void beginInitialization(ProcessContext context, ProcessSession session, FlowFile inputFF, boolean reinitializing) {
    getMetadataRecorder().startFeedInitialization(getFeedId(context, inputFF));
    FlowFile initFF;
    Relationship initRelationship;
    if (context.getProperty(CLONE_INIT_FLOWFILE).asBoolean()) {
        initFF = session.clone(inputFF);
    } else {
        initFF = session.create(inputFF);
    }
    if (reinitializing) {
        boolean useReinit = context.getProperty(USE_REINIT_RELATIONSHIP).asBoolean();
        initRelationship = useReinit ? REL_REINITIALIZE : REL_INITIALIZE;
    } else {
        initRelationship = REL_INITIALIZE;
    }
    initFF = session.putAttribute(initFF, REINITIALIZING_FLAG, Boolean.valueOf(reinitializing).toString());
    session.transfer(initFF, initRelationship);
}

Example 60 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class TestStandardProcessSession method setup.

@Before
public void setup() throws IOException {
    resourceClaimManager = new StandardResourceClaimManager();
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestStandardProcessSession.class.getResource("/conf/nifi.properties").getFile());
    final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class);
    final CounterRepository counterRepo = Mockito.mock(CounterRepository.class);
    provenanceRepo = new MockProvenanceRepository();
    final Connection connection = createConnection();
    final List<Connection> connList = new ArrayList<>();
    connList.add(connection);
    final ProcessGroup procGroup = Mockito.mock(ProcessGroup.class);
    when(procGroup.getIdentifier()).thenReturn("proc-group-identifier-1");
    connectable = Mockito.mock(Connectable.class);
    when(connectable.hasIncomingConnection()).thenReturn(true);
    when(connectable.getIncomingConnections()).thenReturn(connList);
    when(connectable.getProcessGroup()).thenReturn(procGroup);
    when(connectable.getIdentifier()).thenReturn("connectable-1");
    when(connectable.getConnectableType()).thenReturn(ConnectableType.INPUT_PORT);
    when(connectable.getComponentType()).thenReturn("Unit Test Component");
    Mockito.doAnswer(new Answer<Set<Connection>>() {

        @Override
        public Set<Connection> answer(final InvocationOnMock invocation) throws Throwable {
            final Object[] arguments = invocation.getArguments();
            final Relationship relationship = (Relationship) arguments[0];
            if (relationship == Relationship.SELF) {
                return Collections.emptySet();
            } else if (relationship == FAKE_RELATIONSHIP || relationship.equals(FAKE_RELATIONSHIP)) {
                return null;
            } else {
                return new HashSet<>(connList);
            }
        }
    }).when(connectable).getConnections(Mockito.any(Relationship.class));
    when(connectable.getConnections()).thenReturn(new HashSet<>(connList));
    contentRepo = new MockContentRepository();
    contentRepo.initialize(new StandardResourceClaimManager());
    flowFileRepo = new MockFlowFileRepository();
    context = new RepositoryContext(connectable, new AtomicLong(0L), contentRepo, flowFileRepo, flowFileEventRepo, counterRepo, provenanceRepo);
    session = new StandardProcessSession(context, () -> false);
}