Examples with ProcessorConfigDTO org.apache.nifi.web.api.dto.ProcessorConfigDTO used on opensource projects

Example 11 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class StandardFlowSynchronizer method updateProcessor.

private void updateProcessor(final ProcessorNode procNode, final ProcessorDTO processorDTO, final ProcessGroup processGroup, final FlowController controller) throws ProcessorInstantiationException {
    final ProcessorConfigDTO config = processorDTO.getConfig();
    procNode.setProcessGroup(processGroup);
    procNode.setLossTolerant(config.isLossTolerant());
    procNode.setPenalizationPeriod(config.getPenaltyDuration());
    procNode.setYieldPeriod(config.getYieldDuration());
    procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
    updateNonFingerprintedProcessorSettings(procNode, processorDTO);
    if (config.getSchedulingStrategy() != null) {
        procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
    }
    if (config.getExecutionNode() != null) {
        procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
    }
    // must set scheduling strategy before these two
    procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
    procNode.setScheduldingPeriod(config.getSchedulingPeriod());
    if (config.getRunDurationMillis() != null) {
        procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
    }
    procNode.setAnnotationData(config.getAnnotationData());
    if (config.getAutoTerminatedRelationships() != null) {
        final Set<Relationship> relationships = new HashSet<>();
        for (final String rel : config.getAutoTerminatedRelationships()) {
            relationships.add(procNode.getRelationship(rel));
        }
        procNode.setAutoTerminatedRelationships(relationships);
    }
    procNode.setProperties(config.getProperties());
    final ScheduledState scheduledState = ScheduledState.valueOf(processorDTO.getState());
    if (ScheduledState.RUNNING.equals(scheduledState)) {
        controller.startProcessor(processGroup.getIdentifier(), procNode.getIdentifier());
    } else if (ScheduledState.DISABLED.equals(scheduledState)) {
        processGroup.disableProcessor(procNode);
    } else if (ScheduledState.STOPPED.equals(scheduledState)) {
        controller.stopProcessor(processGroup.getIdentifier(), procNode.getIdentifier());
    }
}

Example 12 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class TemplateUtils method scrubProcessors.

/**
 * Scrubs processors prior to saving. This includes removing sensitive properties, validation errors, property descriptors, etc.
 *
 * @param processors procs
 */
private static void scrubProcessors(final Set<ProcessorDTO> processors) {
    // go through each processor
    for (final ProcessorDTO processorDTO : processors) {
        final ProcessorConfigDTO processorConfig = processorDTO.getConfig();
        // ensure that some property configuration have been specified
        if (processorConfig != null) {
            // if properties have been specified, remove sensitive ones
            if (processorConfig.getProperties() != null) {
                Map<String, String> processorProperties = processorConfig.getProperties();
                // look for sensitive properties and remove them
                if (processorConfig.getDescriptors() != null) {
                    final Collection<PropertyDescriptorDTO> descriptors = processorConfig.getDescriptors().values();
                    for (PropertyDescriptorDTO descriptor : descriptors) {
                        if (Boolean.TRUE.equals(descriptor.isSensitive())) {
                            processorProperties.put(descriptor.getName(), null);
                        }
                        scrubPropertyDescriptor(descriptor);
                    }
                }
            }
            processorConfig.setCustomUiUrl(null);
            processorConfig.setDefaultConcurrentTasks(null);
            processorConfig.setDefaultSchedulingPeriod(null);
            processorConfig.setAutoTerminatedRelationships(null);
        }
        if (processorDTO.getRelationships() != null) {
            for (final RelationshipDTO relationship : processorDTO.getRelationships()) {
                relationship.setDescription(null);
            }
        }
        processorDTO.setExtensionMissing(null);
        processorDTO.setMultipleVersionsAvailable(null);
        processorDTO.setValidationErrors(null);
        processorDTO.setInputRequirement(null);
        processorDTO.setDescription(null);
        processorDTO.setInputRequirement(null);
        processorDTO.setPersistsState(null);
        processorDTO.setSupportsBatching(null);
        processorDTO.setSupportsEventDriven(null);
        processorDTO.setSupportsParallelProcessing(null);
    }
}

Example 13 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class FlowFromDOMFactory method getProcessor.

public static ProcessorDTO getProcessor(final Element element, final StringEncryptor encryptor) {
    final ProcessorDTO dto = new ProcessorDTO();
    dto.setId(getString(element, "id"));
    dto.setVersionedComponentId(getString(element, "versionedComponentId"));
    dto.setName(getString(element, "name"));
    dto.setType(getString(element, "class"));
    dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
    dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
    dto.setStyle(getStyle(DomUtils.getChild(element, "styles")));
    final ProcessorConfigDTO configDto = new ProcessorConfigDTO();
    dto.setConfig(configDto);
    configDto.setComments(getString(element, "comment"));
    configDto.setConcurrentlySchedulableTaskCount(getInt(element, "maxConcurrentTasks"));
    final String schedulingPeriod = getString(element, "schedulingPeriod");
    configDto.setSchedulingPeriod(schedulingPeriod);
    configDto.setPenaltyDuration(getString(element, "penalizationPeriod"));
    configDto.setYieldDuration(getString(element, "yieldPeriod"));
    configDto.setBulletinLevel(getString(element, "bulletinLevel"));
    configDto.setLossTolerant(getBoolean(element, "lossTolerant"));
    final ScheduledState scheduledState = getScheduledState(element);
    dto.setState(scheduledState.toString());
    // handle scheduling strategy
    final String schedulingStrategyName = getString(element, "schedulingStrategy");
    if (schedulingStrategyName == null || schedulingStrategyName.trim().isEmpty()) {
        configDto.setSchedulingStrategy(SchedulingStrategy.TIMER_DRIVEN.name());
    } else {
        configDto.setSchedulingStrategy(schedulingStrategyName.trim());
    }
    // handle execution node
    final String executionNode = getString(element, "executionNode");
    if (executionNode == null || executionNode.trim().isEmpty()) {
        configDto.setExecutionNode(ExecutionNode.ALL.name());
    } else {
        configDto.setExecutionNode(executionNode.trim());
    }
    final Long runDurationNanos = getOptionalLong(element, "runDurationNanos");
    if (runDurationNanos != null) {
        configDto.setRunDurationMillis(TimeUnit.NANOSECONDS.toMillis(runDurationNanos));
    }
    configDto.setProperties(getProperties(element, encryptor));
    configDto.setAnnotationData(getString(element, "annotationData"));
    final Set<String> autoTerminatedRelationships = new HashSet<>();
    final List<Element> autoTerminateList = getChildrenByTagName(element, "autoTerminatedRelationship");
    for (final Element autoTerminateElement : autoTerminateList) {
        autoTerminatedRelationships.add(autoTerminateElement.getTextContent());
    }
    configDto.setAutoTerminatedRelationships(autoTerminatedRelationships);
    return dto;
}

Example 14 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 15 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi-minifi by apache.

the class ProcessorSchemaTest method setup.

@Before
public void setup() {
    config = new ProcessorConfigDTO();
    RelationshipDTO relationshipDTO = new RelationshipDTO();
    relationshipDTO.setName(testRelationship);
    relationshipDTO.setAutoTerminate(true);
    dto = new ProcessorDTO();
    dto.setConfig(config);
    dto.setName(testName);
    dto.setId(testId);
    dto.setType(testProcessorClass);
    config.setSchedulingStrategy(testSchedulingStrategy);
    config.setSchedulingPeriod(testSchedulingPeriod);
    config.setConcurrentlySchedulableTaskCount(testMaxConcurrentTasks);
    config.setPenaltyDuration(testPenalizationPeriod);
    config.setYieldDuration(testYieldDuration);
    config.setRunDurationMillis(testRunDurationNanos / 1000);
    config.setAnnotationData(testAnnotationData);
    dto.setRelationships(Arrays.asList(relationshipDTO));
    Map<String, String> properties = new HashMap<>();
    properties.put(testKey, testValue);
    config.setProperties(properties);
    map = new HashMap<>();
    map.put(CommonPropertyKeys.NAME_KEY, testName);
    map.put(CommonPropertyKeys.ID_KEY, testId);
    map.put(CLASS_KEY, testProcessorClass);
    map.put(CommonPropertyKeys.SCHEDULING_STRATEGY_KEY, testSchedulingStrategy);
    map.put(CommonPropertyKeys.SCHEDULING_PERIOD_KEY, testSchedulingPeriod);
    map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, testMaxConcurrentTasks);
    map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, testPenalizationPeriod);
    map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, testYieldDuration);
    map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, testRunDurationNanos);
    map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, Arrays.asList(testRelationship));
    map.put(PROPERTIES_KEY, new HashMap<>(properties));
    map.put(ANNOTATION_DATA_KEY, testAnnotationData);
}