Examples with FlowFilePrioritizer org.apache.nifi.flowfile.FlowFilePrioritizer used on opensource projects

Example 6 with FlowFilePrioritizer

use of org.apache.nifi.flowfile.FlowFilePrioritizer in project nifi by apache.

the class StandardFlowSynchronizer method addProcessGroup.

private ProcessGroup addProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, final StringEncryptor encryptor, final FlowEncodingVersion encodingVersion) throws ProcessorInstantiationException {
    // get the parent group ID
    final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier();
    // add the process group
    final ProcessGroupDTO processGroupDTO = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion);
    final ProcessGroup processGroup = controller.createProcessGroup(processGroupDTO.getId());
    processGroup.setComments(processGroupDTO.getComments());
    processGroup.setVersionedComponentId(processGroupDTO.getVersionedComponentId());
    processGroup.setPosition(toPosition(processGroupDTO.getPosition()));
    processGroup.setName(processGroupDTO.getName());
    processGroup.setParent(parentGroup);
    if (parentGroup == null) {
        controller.setRootGroup(processGroup);
    } else {
        parentGroup.addProcessGroup(processGroup);
    }
    // Set the variables for the variable registry
    final Map<String, String> variables = new HashMap<>();
    final List<Element> variableElements = getChildrenByTagName(processGroupElement, "variable");
    for (final Element variableElement : variableElements) {
        final String variableName = variableElement.getAttribute("name");
        final String variableValue = variableElement.getAttribute("value");
        if (variableName == null || variableValue == null) {
            continue;
        }
        variables.put(variableName, variableValue);
    }
    processGroup.setVariables(variables);
    final VersionControlInformationDTO versionControlInfoDto = processGroupDTO.getVersionControlInformation();
    if (versionControlInfoDto != null) {
        final FlowRegistry flowRegistry = controller.getFlowRegistryClient().getFlowRegistry(versionControlInfoDto.getRegistryId());
        final String registryName = flowRegistry == null ? versionControlInfoDto.getRegistryId() : flowRegistry.getName();
        versionControlInfoDto.setState(VersionedFlowState.SYNC_FAILURE.name());
        versionControlInfoDto.setStateExplanation("Process Group has not yet been synchronized with the Flow Registry");
        final StandardVersionControlInformation versionControlInformation = StandardVersionControlInformation.Builder.fromDto(versionControlInfoDto).registryName(registryName).build();
        // pass empty map for the version control mapping because the VersionedComponentId has already been set on the components
        processGroup.setVersionControlInformation(versionControlInformation, Collections.emptyMap());
    }
    // Add Controller Services
    final List<Element> serviceNodeList = getChildrenByTagName(processGroupElement, "controllerService");
    if (!serviceNodeList.isEmpty()) {
        final Map<ControllerServiceNode, Element> controllerServices = ControllerServiceLoader.loadControllerServices(serviceNodeList, controller, processGroup, encryptor);
        ControllerServiceLoader.enableControllerServices(controllerServices, controller, encryptor, autoResumeState);
    }
    // add processors
    final List<Element> processorNodeList = getChildrenByTagName(processGroupElement, "processor");
    for (final Element processorElement : processorNodeList) {
        final ProcessorDTO processorDTO = FlowFromDOMFactory.getProcessor(processorElement, encryptor);
        BundleCoordinate coordinate;
        try {
            coordinate = BundleUtils.getCompatibleBundle(processorDTO.getType(), processorDTO.getBundle());
        } catch (final IllegalStateException e) {
            final BundleDTO bundleDTO = processorDTO.getBundle();
            if (bundleDTO == null) {
                coordinate = BundleCoordinate.UNKNOWN_COORDINATE;
            } else {
                coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion());
            }
        }
        final ProcessorNode procNode = controller.createProcessor(processorDTO.getType(), processorDTO.getId(), coordinate, false);
        procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
        processGroup.addProcessor(procNode);
        updateProcessor(procNode, processorDTO, processGroup, controller);
    }
    // add input ports
    final List<Element> inputPortNodeList = getChildrenByTagName(processGroupElement, "inputPort");
    for (final Element inputPortElement : inputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(inputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteInputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalInputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addInputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableInputPort(port);
        }
    }
    // add output ports
    final List<Element> outputPortNodeList = getChildrenByTagName(processGroupElement, "outputPort");
    for (final Element outputPortElement : outputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(outputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteOutputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalOutputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addOutputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableOutputPort(port);
        }
    }
    // add funnels
    final List<Element> funnelNodeList = getChildrenByTagName(processGroupElement, "funnel");
    for (final Element funnelElement : funnelNodeList) {
        final FunnelDTO funnelDTO = FlowFromDOMFactory.getFunnel(funnelElement);
        final Funnel funnel = controller.createFunnel(funnelDTO.getId());
        funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
        funnel.setPosition(toPosition(funnelDTO.getPosition()));
        // Since this is called during startup, we want to add the funnel without enabling it
        // and then tell the controller to enable it. This way, if the controller is not fully
        // initialized, the starting of the funnel is delayed until the controller is ready.
        processGroup.addFunnel(funnel, false);
        controller.startConnectable(funnel);
    }
    // add labels
    final List<Element> labelNodeList = getChildrenByTagName(processGroupElement, "label");
    for (final Element labelElement : labelNodeList) {
        final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement);
        final Label label = controller.createLabel(labelDTO.getId(), labelDTO.getLabel());
        label.setVersionedComponentId(labelDTO.getVersionedComponentId());
        label.setStyle(labelDTO.getStyle());
        label.setPosition(toPosition(labelDTO.getPosition()));
        label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
        processGroup.addLabel(label);
    }
    // add nested process groups (recursively)
    final List<Element> nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup");
    for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) {
        addProcessGroup(controller, processGroup, nestedProcessGroupElement, encryptor, encodingVersion);
    }
    // add remote process group
    final List<Element> remoteProcessGroupNodeList = getChildrenByTagName(processGroupElement, "remoteProcessGroup");
    for (final Element remoteProcessGroupElement : remoteProcessGroupNodeList) {
        final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteProcessGroupElement, encryptor);
        final RemoteProcessGroup remoteGroup = controller.createRemoteProcessGroup(remoteGroupDto.getId(), remoteGroupDto.getTargetUris());
        remoteGroup.setVersionedComponentId(remoteGroupDto.getVersionedComponentId());
        remoteGroup.setComments(remoteGroupDto.getComments());
        remoteGroup.setPosition(toPosition(remoteGroupDto.getPosition()));
        final String name = remoteGroupDto.getName();
        if (name != null && !name.trim().isEmpty()) {
            remoteGroup.setName(name);
        }
        remoteGroup.setProcessGroup(processGroup);
        remoteGroup.setCommunicationsTimeout(remoteGroupDto.getCommunicationsTimeout());
        if (remoteGroupDto.getYieldDuration() != null) {
            remoteGroup.setYieldDuration(remoteGroupDto.getYieldDuration());
        }
        final String transportProtocol = remoteGroupDto.getTransportProtocol();
        if (transportProtocol != null && !transportProtocol.trim().isEmpty()) {
            remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(transportProtocol.toUpperCase()));
        }
        if (remoteGroupDto.getProxyHost() != null) {
            remoteGroup.setProxyHost(remoteGroupDto.getProxyHost());
        }
        if (remoteGroupDto.getProxyPort() != null) {
            remoteGroup.setProxyPort(remoteGroupDto.getProxyPort());
        }
        if (remoteGroupDto.getProxyUser() != null) {
            remoteGroup.setProxyUser(remoteGroupDto.getProxyUser());
        }
        if (remoteGroupDto.getProxyPassword() != null) {
            remoteGroup.setProxyPassword(remoteGroupDto.getProxyPassword());
        }
        if (StringUtils.isBlank(remoteGroupDto.getLocalNetworkInterface())) {
            remoteGroup.setNetworkInterface(null);
        } else {
            remoteGroup.setNetworkInterface(remoteGroupDto.getLocalNetworkInterface());
        }
        final Set<RemoteProcessGroupPortDescriptor> inputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "inputPort")) {
            inputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setInputPorts(inputPorts, false);
        final Set<RemoteProcessGroupPortDescriptor> outputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "outputPort")) {
            outputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setOutputPorts(outputPorts, false);
        processGroup.addRemoteProcessGroup(remoteGroup);
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : outputPorts) {
            final RemoteGroupPort port = remoteGroup.getOutputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : inputPorts) {
            final RemoteGroupPort port = remoteGroup.getInputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
    }
    // add connections
    final List<Element> connectionNodeList = getChildrenByTagName(processGroupElement, "connection");
    for (final Element connectionElement : connectionNodeList) {
        final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement);
        final Connectable source;
        final ConnectableDTO sourceDto = dto.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(sourceDto.getGroupId());
            source = remoteGroup.getOutputPort(sourceDto.getId());
        } else {
            final ProcessGroup sourceGroup = controller.getGroup(sourceDto.getGroupId());
            if (sourceGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Source: " + dto.getSource().getGroupId());
            }
            source = sourceGroup.getConnectable(sourceDto.getId());
        }
        if (source == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Source: " + dto.getSource().getId());
        }
        final Connectable destination;
        final ConnectableDTO destinationDto = dto.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(destinationDto.getGroupId());
            destination = remoteGroup.getInputPort(destinationDto.getId());
        } else {
            final ProcessGroup destinationGroup = controller.getGroup(destinationDto.getGroupId());
            if (destinationGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Destination: " + dto.getDestination().getGroupId());
            }
            destination = destinationGroup.getConnectable(destinationDto.getId());
        }
        if (destination == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Destination: " + dto.getDestination().getId());
        }
        final Connection connection = controller.createConnection(dto.getId(), dto.getName(), source, destination, dto.getSelectedRelationships());
        connection.setVersionedComponentId(dto.getVersionedComponentId());
        connection.setProcessGroup(processGroup);
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : dto.getBends()) {
            bendPoints.add(new Position(bend.getX(), bend.getY()));
        }
        connection.setBendPoints(bendPoints);
        final Long zIndex = dto.getzIndex();
        if (zIndex != null) {
            connection.setZIndex(zIndex);
        }
        if (dto.getLabelIndex() != null) {
            connection.setLabelIndex(dto.getLabelIndex());
        }
        List<FlowFilePrioritizer> newPrioritizers = null;
        final List<String> prioritizers = dto.getPrioritizers();
        if (prioritizers != null) {
            final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
            newPrioritizers = new ArrayList<>();
            for (final String className : newPrioritizersClasses) {
                try {
                    newPrioritizers.add(controller.createPrioritizer(className));
                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                    throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                }
            }
        }
        if (newPrioritizers != null) {
            connection.getFlowFileQueue().setPriorities(newPrioritizers);
        }
        if (dto.getBackPressureObjectThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold());
        }
        if (dto.getBackPressureDataSizeThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold());
        }
        if (dto.getFlowFileExpiration() != null) {
            connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration());
        }
        processGroup.addConnection(connection);
    }
    final List<Element> templateNodeList = getChildrenByTagName(processGroupElement, "template");
    for (final Element templateNode : templateNodeList) {
        final TemplateDTO templateDTO = TemplateUtils.parseDto(templateNode);
        final Template template = new Template(templateDTO);
        processGroup.addTemplate(template);
    }
    return processGroup;
}

Example 7 with FlowFilePrioritizer

use of org.apache.nifi.flowfile.FlowFilePrioritizer in project nifi by apache.

the class StandardFlowSerializer method addConnection.

private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "versionedComponentId", connection.getVersionedComponentId());
    addTextElement(element, "name", connection.getName());
    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
        addPosition(bendPointsElement, bendPoint, "bendPoint");
    }
    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());
    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
        sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
        sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }
    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
        destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
        destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }
    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());
    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());
    for (final Relationship relationship : connection.getRelationships()) {
        addTextElement(element, "relationship", relationship.getName());
    }
    addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());
    addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
        final String className = comparator.getClass().getCanonicalName();
        addTextElement(element, "queuePrioritizerClass", className);
    }
    parentElement.appendChild(element);
}

Example 8 with FlowFilePrioritizer

use of org.apache.nifi.flowfile.FlowFilePrioritizer in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 9 with FlowFilePrioritizer

use of org.apache.nifi.flowfile.FlowFilePrioritizer in project nifi by apache.

the class StandardProcessGroup method updateConnection.

private void updateConnection(final Connection connection, final VersionedConnection proposed) {
    connection.setBendPoints(proposed.getBends() == null ? Collections.emptyList() : proposed.getBends().stream().map(pos -> new Position(pos.getX(), pos.getY())).collect(Collectors.toList()));
    connection.setDestination(getConnectable(connection.getProcessGroup(), proposed.getDestination()));
    connection.setLabelIndex(proposed.getLabelIndex());
    connection.setName(proposed.getName());
    connection.setRelationships(proposed.getSelectedRelationships().stream().map(name -> new Relationship.Builder().name(name).build()).collect(Collectors.toSet()));
    connection.setZIndex(proposed.getzIndex());
    final FlowFileQueue queue = connection.getFlowFileQueue();
    queue.setBackPressureDataSizeThreshold(proposed.getBackPressureDataSizeThreshold());
    queue.setBackPressureObjectThreshold(proposed.getBackPressureObjectThreshold());
    queue.setFlowFileExpiration(proposed.getFlowFileExpiration());
    final List<FlowFilePrioritizer> prioritizers = proposed.getPrioritizers() == null ? Collections.emptyList() : proposed.getPrioritizers().stream().map(prioritizerName -> {
        try {
            return flowController.createPrioritizer(prioritizerName);
        } catch (final Exception e) {
            throw new IllegalStateException("Failed to create Prioritizer of type " + prioritizerName + " for Connection with ID " + connection.getIdentifier());
        }
    }).collect(Collectors.toList());
    queue.setPriorities(prioritizers);
}

Example 10 with FlowFilePrioritizer

use of org.apache.nifi.flowfile.FlowFilePrioritizer in project nifi by apache.

the class FlowController method createPrioritizer.

public FlowFilePrioritizer createPrioritizer(final String type) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    FlowFilePrioritizer prioritizer;
    final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        final List<Bundle> prioritizerBundles = ExtensionManager.getBundles(type);
        if (prioritizerBundles.size() == 0) {
            throw new IllegalStateException(String.format("The specified class '%s' is not known to this nifi.", type));
        }
        if (prioritizerBundles.size() > 1) {
            throw new IllegalStateException(String.format("Multiple bundles found for the specified class '%s', only one is allowed.", type));
        }
        final Bundle bundle = prioritizerBundles.get(0);
        final ClassLoader detectedClassLoaderForType = bundle.getClassLoader();
        final Class<?> rawClass = Class.forName(type, true, detectedClassLoaderForType);
        Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
        final Class<? extends FlowFilePrioritizer> prioritizerClass = rawClass.asSubclass(FlowFilePrioritizer.class);
        final Object processorObj = prioritizerClass.newInstance();
        prioritizer = prioritizerClass.cast(processorObj);
        return prioritizer;
    } finally {
        if (ctxClassLoader != null) {
            Thread.currentThread().setContextClassLoader(ctxClassLoader);
        }
    }
}