Examples with TemplateDTO org.apache.nifi.web.api.dto.TemplateDTO used on opensource projects

Example 11 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project nifi by apache.

the class StandardFlowSynchronizer method updateProcessGroup.

private ProcessGroup updateProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, final StringEncryptor encryptor, final FlowEncodingVersion encodingVersion) throws ProcessorInstantiationException {
    // get the parent group ID
    final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier();
    // get the process group
    final ProcessGroupDTO processGroupDto = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion);
    // update the process group
    if (parentId == null) {
        /*
             * Labels are not included in the "inherit flow" algorithm, so we cannot
             * blindly update them because they may not exist in the current flow.
             * Therefore, we first remove all labels, and then let the updating
             * process add labels defined in the new flow.
             */
        final ProcessGroup root = controller.getGroup(controller.getRootGroupId());
        for (final Label label : root.findAllLabels()) {
            label.getProcessGroup().removeLabel(label);
        }
    }
    // update the process group
    controller.updateProcessGroup(processGroupDto);
    // get the real process group and ID
    final ProcessGroup processGroup = controller.getGroup(processGroupDto.getId());
    // determine the scheduled state of all of the Controller Service
    final List<Element> controllerServiceNodeList = getChildrenByTagName(processGroupElement, "controllerService");
    final Set<ControllerServiceNode> toDisable = new HashSet<>();
    final Set<ControllerServiceNode> toEnable = new HashSet<>();
    for (final Element serviceElement : controllerServiceNodeList) {
        final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(serviceElement, encryptor);
        final ControllerServiceNode serviceNode = processGroup.getControllerService(dto.getId());
        // Check if the controller service is in the correct state. We consider it the correct state if
        // we are in a transitional state and heading in the right direction or already in the correct state.
        // E.g., it is the correct state if it should be 'DISABLED' and it is either DISABLED or DISABLING.
        final ControllerServiceState serviceState = getFinalTransitionState(serviceNode.getState());
        final ControllerServiceState clusterState = getFinalTransitionState(ControllerServiceState.valueOf(dto.getState()));
        if (serviceState != clusterState) {
            switch(clusterState) {
                case DISABLED:
                    toDisable.add(serviceNode);
                    break;
                case ENABLED:
                    toEnable.add(serviceNode);
                    break;
            }
        }
    }
    controller.disableControllerServicesAsync(toDisable);
    controller.enableControllerServices(toEnable);
    // processors & ports cannot be updated - they must be the same. Except for the scheduled state.
    final List<Element> processorNodeList = getChildrenByTagName(processGroupElement, "processor");
    for (final Element processorElement : processorNodeList) {
        final ProcessorDTO dto = FlowFromDOMFactory.getProcessor(processorElement, encryptor);
        final ProcessorNode procNode = processGroup.getProcessor(dto.getId());
        updateNonFingerprintedProcessorSettings(procNode, dto);
        if (!procNode.getScheduledState().name().equals(dto.getState())) {
            try {
                switch(ScheduledState.valueOf(dto.getState())) {
                    case DISABLED:
                        // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing),
                        // and then we have to disable it.
                        controller.stopProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier());
                        procNode.getProcessGroup().disableProcessor(procNode);
                        break;
                    case RUNNING:
                        // we want to run now. Make sure processor is not disabled and then start it.
                        procNode.getProcessGroup().enableProcessor(procNode);
                        controller.startProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier(), false);
                        break;
                    case STOPPED:
                        if (procNode.getScheduledState() == ScheduledState.DISABLED) {
                            procNode.getProcessGroup().enableProcessor(procNode);
                        } else if (procNode.getScheduledState() == ScheduledState.RUNNING) {
                            controller.stopProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier());
                        }
                        break;
                }
            } catch (final IllegalStateException ise) {
                logger.error("Failed to change Scheduled State of {} from {} to {} due to {}", procNode, procNode.getScheduledState().name(), dto.getState(), ise.toString());
                logger.error("", ise);
                // create bulletin for the Processor Node
                controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin(procNode, "Node Reconnection", Severity.ERROR.name(), "Failed to change Scheduled State of " + procNode + " from " + procNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString()));
                // create bulletin at Controller level.
                controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), "Failed to change Scheduled State of " + procNode + " from " + procNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString()));
            }
        }
    }
    final List<Element> inputPortList = getChildrenByTagName(processGroupElement, "inputPort");
    for (final Element portElement : inputPortList) {
        final PortDTO dto = FlowFromDOMFactory.getPort(portElement);
        final Port port = processGroup.getInputPort(dto.getId());
        if (!port.getScheduledState().name().equals(dto.getState())) {
            switch(ScheduledState.valueOf(dto.getState())) {
                case DISABLED:
                    // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing),
                    // and then we have to disable it.
                    controller.stopConnectable(port);
                    port.getProcessGroup().disableInputPort(port);
                    break;
                case RUNNING:
                    // we want to run now. Make sure processor is not disabled and then start it.
                    port.getProcessGroup().enableInputPort(port);
                    controller.startConnectable(port);
                    break;
                case STOPPED:
                    if (port.getScheduledState() == ScheduledState.DISABLED) {
                        port.getProcessGroup().enableInputPort(port);
                    } else if (port.getScheduledState() == ScheduledState.RUNNING) {
                        controller.stopConnectable(port);
                    }
                    break;
            }
        }
    }
    final List<Element> outputPortList = getChildrenByTagName(processGroupElement, "outputPort");
    for (final Element portElement : outputPortList) {
        final PortDTO dto = FlowFromDOMFactory.getPort(portElement);
        final Port port = processGroup.getOutputPort(dto.getId());
        if (!port.getScheduledState().name().equals(dto.getState())) {
            switch(ScheduledState.valueOf(dto.getState())) {
                case DISABLED:
                    // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing),
                    // and then we have to disable it.
                    controller.stopConnectable(port);
                    port.getProcessGroup().disableOutputPort(port);
                    break;
                case RUNNING:
                    // we want to run now. Make sure processor is not disabled and then start it.
                    port.getProcessGroup().enableOutputPort(port);
                    controller.startConnectable(port);
                    break;
                case STOPPED:
                    if (port.getScheduledState() == ScheduledState.DISABLED) {
                        port.getProcessGroup().enableOutputPort(port);
                    } else if (port.getScheduledState() == ScheduledState.RUNNING) {
                        controller.stopConnectable(port);
                    }
                    break;
            }
        }
    }
    // Update scheduled state of Remote Group Ports
    final List<Element> remoteProcessGroupList = getChildrenByTagName(processGroupElement, "remoteProcessGroup");
    for (final Element remoteGroupElement : remoteProcessGroupList) {
        final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteGroupElement, encryptor);
        final RemoteProcessGroup rpg = processGroup.getRemoteProcessGroup(remoteGroupDto.getId());
        // input ports
        final List<Element> inputPortElements = getChildrenByTagName(remoteGroupElement, "inputPort");
        for (final Element inputPortElement : inputPortElements) {
            final RemoteProcessGroupPortDescriptor portDescriptor = FlowFromDOMFactory.getRemoteProcessGroupPort(inputPortElement);
            final String inputPortId = portDescriptor.getId();
            final RemoteGroupPort inputPort = rpg.getInputPort(inputPortId);
            if (inputPort == null) {
                continue;
            }
            if (portDescriptor.isTransmitting()) {
                if (inputPort.getScheduledState() != ScheduledState.RUNNING && inputPort.getScheduledState() != ScheduledState.STARTING) {
                    rpg.startTransmitting(inputPort);
                }
            } else if (inputPort.getScheduledState() != ScheduledState.STOPPED && inputPort.getScheduledState() != ScheduledState.STOPPING) {
                rpg.stopTransmitting(inputPort);
            }
        }
        // output ports
        final List<Element> outputPortElements = getChildrenByTagName(remoteGroupElement, "outputPort");
        for (final Element outputPortElement : outputPortElements) {
            final RemoteProcessGroupPortDescriptor portDescriptor = FlowFromDOMFactory.getRemoteProcessGroupPort(outputPortElement);
            final String outputPortId = portDescriptor.getId();
            final RemoteGroupPort outputPort = rpg.getOutputPort(outputPortId);
            if (outputPort == null) {
                continue;
            }
            if (portDescriptor.isTransmitting()) {
                if (outputPort.getScheduledState() != ScheduledState.RUNNING && outputPort.getScheduledState() != ScheduledState.STARTING) {
                    rpg.startTransmitting(outputPort);
                }
            } else if (outputPort.getScheduledState() != ScheduledState.STOPPED && outputPort.getScheduledState() != ScheduledState.STOPPING) {
                rpg.stopTransmitting(outputPort);
            }
        }
    }
    // add labels
    final List<Element> labelNodeList = getChildrenByTagName(processGroupElement, "label");
    for (final Element labelElement : labelNodeList) {
        final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement);
        final Label label = controller.createLabel(labelDTO.getId(), labelDTO.getLabel());
        label.setStyle(labelDTO.getStyle());
        label.setPosition(new Position(labelDTO.getPosition().getX(), labelDTO.getPosition().getY()));
        label.setVersionedComponentId(labelDTO.getVersionedComponentId());
        if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
            label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
        }
        processGroup.addLabel(label);
    }
    // update nested process groups (recursively)
    final List<Element> nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup");
    for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) {
        updateProcessGroup(controller, processGroup, nestedProcessGroupElement, encryptor, encodingVersion);
    }
    // update connections
    final List<Element> connectionNodeList = getChildrenByTagName(processGroupElement, "connection");
    for (final Element connectionElement : connectionNodeList) {
        final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement);
        final Connection connection = processGroup.getConnection(dto.getId());
        connection.setName(dto.getName());
        connection.setProcessGroup(processGroup);
        if (dto.getLabelIndex() != null) {
            connection.setLabelIndex(dto.getLabelIndex());
        }
        if (dto.getzIndex() != null) {
            connection.setZIndex(dto.getzIndex());
        }
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : dto.getBends()) {
            bendPoints.add(new Position(bend.getX(), bend.getY()));
        }
        connection.setBendPoints(bendPoints);
        List<FlowFilePrioritizer> newPrioritizers = null;
        final List<String> prioritizers = dto.getPrioritizers();
        if (prioritizers != null) {
            final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
            newPrioritizers = new ArrayList<>();
            for (final String className : newPrioritizersClasses) {
                try {
                    newPrioritizers.add(controller.createPrioritizer(className));
                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                    throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                }
            }
        }
        if (newPrioritizers != null) {
            connection.getFlowFileQueue().setPriorities(newPrioritizers);
        }
        if (dto.getBackPressureObjectThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold());
        }
        if (dto.getBackPressureDataSizeThreshold() != null && !dto.getBackPressureDataSizeThreshold().trim().isEmpty()) {
            connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold());
        }
        if (dto.getFlowFileExpiration() != null) {
            connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration());
        }
    }
    // Replace the templates with those from the proposed flow
    final List<Element> templateNodeList = getChildrenByTagName(processGroupElement, "template");
    for (final Element templateElement : templateNodeList) {
        final TemplateDTO templateDto = TemplateUtils.parseDto(templateElement);
        final Template template = new Template(templateDto);
        // This just makes sure that they do.
        if (processGroup.getTemplate(template.getIdentifier()) != null) {
            processGroup.removeTemplate(template);
        }
        processGroup.addTemplate(template);
    }
    return processGroup;
}

Example 12 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project nifi by apache.

the class StandardFlowSynchronizer method addProcessGroup.

private ProcessGroup addProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, final StringEncryptor encryptor, final FlowEncodingVersion encodingVersion) throws ProcessorInstantiationException {
    // get the parent group ID
    final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier();
    // add the process group
    final ProcessGroupDTO processGroupDTO = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion);
    final ProcessGroup processGroup = controller.createProcessGroup(processGroupDTO.getId());
    processGroup.setComments(processGroupDTO.getComments());
    processGroup.setVersionedComponentId(processGroupDTO.getVersionedComponentId());
    processGroup.setPosition(toPosition(processGroupDTO.getPosition()));
    processGroup.setName(processGroupDTO.getName());
    processGroup.setParent(parentGroup);
    if (parentGroup == null) {
        controller.setRootGroup(processGroup);
    } else {
        parentGroup.addProcessGroup(processGroup);
    }
    // Set the variables for the variable registry
    final Map<String, String> variables = new HashMap<>();
    final List<Element> variableElements = getChildrenByTagName(processGroupElement, "variable");
    for (final Element variableElement : variableElements) {
        final String variableName = variableElement.getAttribute("name");
        final String variableValue = variableElement.getAttribute("value");
        if (variableName == null || variableValue == null) {
            continue;
        }
        variables.put(variableName, variableValue);
    }
    processGroup.setVariables(variables);
    final VersionControlInformationDTO versionControlInfoDto = processGroupDTO.getVersionControlInformation();
    if (versionControlInfoDto != null) {
        final FlowRegistry flowRegistry = controller.getFlowRegistryClient().getFlowRegistry(versionControlInfoDto.getRegistryId());
        final String registryName = flowRegistry == null ? versionControlInfoDto.getRegistryId() : flowRegistry.getName();
        versionControlInfoDto.setState(VersionedFlowState.SYNC_FAILURE.name());
        versionControlInfoDto.setStateExplanation("Process Group has not yet been synchronized with the Flow Registry");
        final StandardVersionControlInformation versionControlInformation = StandardVersionControlInformation.Builder.fromDto(versionControlInfoDto).registryName(registryName).build();
        // pass empty map for the version control mapping because the VersionedComponentId has already been set on the components
        processGroup.setVersionControlInformation(versionControlInformation, Collections.emptyMap());
    }
    // Add Controller Services
    final List<Element> serviceNodeList = getChildrenByTagName(processGroupElement, "controllerService");
    if (!serviceNodeList.isEmpty()) {
        final Map<ControllerServiceNode, Element> controllerServices = ControllerServiceLoader.loadControllerServices(serviceNodeList, controller, processGroup, encryptor);
        ControllerServiceLoader.enableControllerServices(controllerServices, controller, encryptor, autoResumeState);
    }
    // add processors
    final List<Element> processorNodeList = getChildrenByTagName(processGroupElement, "processor");
    for (final Element processorElement : processorNodeList) {
        final ProcessorDTO processorDTO = FlowFromDOMFactory.getProcessor(processorElement, encryptor);
        BundleCoordinate coordinate;
        try {
            coordinate = BundleUtils.getCompatibleBundle(processorDTO.getType(), processorDTO.getBundle());
        } catch (final IllegalStateException e) {
            final BundleDTO bundleDTO = processorDTO.getBundle();
            if (bundleDTO == null) {
                coordinate = BundleCoordinate.UNKNOWN_COORDINATE;
            } else {
                coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion());
            }
        }
        final ProcessorNode procNode = controller.createProcessor(processorDTO.getType(), processorDTO.getId(), coordinate, false);
        procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
        processGroup.addProcessor(procNode);
        updateProcessor(procNode, processorDTO, processGroup, controller);
    }
    // add input ports
    final List<Element> inputPortNodeList = getChildrenByTagName(processGroupElement, "inputPort");
    for (final Element inputPortElement : inputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(inputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteInputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalInputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addInputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableInputPort(port);
        }
    }
    // add output ports
    final List<Element> outputPortNodeList = getChildrenByTagName(processGroupElement, "outputPort");
    for (final Element outputPortElement : outputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(outputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteOutputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalOutputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addOutputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableOutputPort(port);
        }
    }
    // add funnels
    final List<Element> funnelNodeList = getChildrenByTagName(processGroupElement, "funnel");
    for (final Element funnelElement : funnelNodeList) {
        final FunnelDTO funnelDTO = FlowFromDOMFactory.getFunnel(funnelElement);
        final Funnel funnel = controller.createFunnel(funnelDTO.getId());
        funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
        funnel.setPosition(toPosition(funnelDTO.getPosition()));
        // Since this is called during startup, we want to add the funnel without enabling it
        // and then tell the controller to enable it. This way, if the controller is not fully
        // initialized, the starting of the funnel is delayed until the controller is ready.
        processGroup.addFunnel(funnel, false);
        controller.startConnectable(funnel);
    }
    // add labels
    final List<Element> labelNodeList = getChildrenByTagName(processGroupElement, "label");
    for (final Element labelElement : labelNodeList) {
        final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement);
        final Label label = controller.createLabel(labelDTO.getId(), labelDTO.getLabel());
        label.setVersionedComponentId(labelDTO.getVersionedComponentId());
        label.setStyle(labelDTO.getStyle());
        label.setPosition(toPosition(labelDTO.getPosition()));
        label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
        processGroup.addLabel(label);
    }
    // add nested process groups (recursively)
    final List<Element> nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup");
    for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) {
        addProcessGroup(controller, processGroup, nestedProcessGroupElement, encryptor, encodingVersion);
    }
    // add remote process group
    final List<Element> remoteProcessGroupNodeList = getChildrenByTagName(processGroupElement, "remoteProcessGroup");
    for (final Element remoteProcessGroupElement : remoteProcessGroupNodeList) {
        final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteProcessGroupElement, encryptor);
        final RemoteProcessGroup remoteGroup = controller.createRemoteProcessGroup(remoteGroupDto.getId(), remoteGroupDto.getTargetUris());
        remoteGroup.setVersionedComponentId(remoteGroupDto.getVersionedComponentId());
        remoteGroup.setComments(remoteGroupDto.getComments());
        remoteGroup.setPosition(toPosition(remoteGroupDto.getPosition()));
        final String name = remoteGroupDto.getName();
        if (name != null && !name.trim().isEmpty()) {
            remoteGroup.setName(name);
        }
        remoteGroup.setProcessGroup(processGroup);
        remoteGroup.setCommunicationsTimeout(remoteGroupDto.getCommunicationsTimeout());
        if (remoteGroupDto.getYieldDuration() != null) {
            remoteGroup.setYieldDuration(remoteGroupDto.getYieldDuration());
        }
        final String transportProtocol = remoteGroupDto.getTransportProtocol();
        if (transportProtocol != null && !transportProtocol.trim().isEmpty()) {
            remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(transportProtocol.toUpperCase()));
        }
        if (remoteGroupDto.getProxyHost() != null) {
            remoteGroup.setProxyHost(remoteGroupDto.getProxyHost());
        }
        if (remoteGroupDto.getProxyPort() != null) {
            remoteGroup.setProxyPort(remoteGroupDto.getProxyPort());
        }
        if (remoteGroupDto.getProxyUser() != null) {
            remoteGroup.setProxyUser(remoteGroupDto.getProxyUser());
        }
        if (remoteGroupDto.getProxyPassword() != null) {
            remoteGroup.setProxyPassword(remoteGroupDto.getProxyPassword());
        }
        if (StringUtils.isBlank(remoteGroupDto.getLocalNetworkInterface())) {
            remoteGroup.setNetworkInterface(null);
        } else {
            remoteGroup.setNetworkInterface(remoteGroupDto.getLocalNetworkInterface());
        }
        final Set<RemoteProcessGroupPortDescriptor> inputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "inputPort")) {
            inputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setInputPorts(inputPorts, false);
        final Set<RemoteProcessGroupPortDescriptor> outputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "outputPort")) {
            outputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setOutputPorts(outputPorts, false);
        processGroup.addRemoteProcessGroup(remoteGroup);
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : outputPorts) {
            final RemoteGroupPort port = remoteGroup.getOutputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : inputPorts) {
            final RemoteGroupPort port = remoteGroup.getInputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
    }
    // add connections
    final List<Element> connectionNodeList = getChildrenByTagName(processGroupElement, "connection");
    for (final Element connectionElement : connectionNodeList) {
        final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement);
        final Connectable source;
        final ConnectableDTO sourceDto = dto.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(sourceDto.getGroupId());
            source = remoteGroup.getOutputPort(sourceDto.getId());
        } else {
            final ProcessGroup sourceGroup = controller.getGroup(sourceDto.getGroupId());
            if (sourceGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Source: " + dto.getSource().getGroupId());
            }
            source = sourceGroup.getConnectable(sourceDto.getId());
        }
        if (source == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Source: " + dto.getSource().getId());
        }
        final Connectable destination;
        final ConnectableDTO destinationDto = dto.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(destinationDto.getGroupId());
            destination = remoteGroup.getInputPort(destinationDto.getId());
        } else {
            final ProcessGroup destinationGroup = controller.getGroup(destinationDto.getGroupId());
            if (destinationGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Destination: " + dto.getDestination().getGroupId());
            }
            destination = destinationGroup.getConnectable(destinationDto.getId());
        }
        if (destination == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Destination: " + dto.getDestination().getId());
        }
        final Connection connection = controller.createConnection(dto.getId(), dto.getName(), source, destination, dto.getSelectedRelationships());
        connection.setVersionedComponentId(dto.getVersionedComponentId());
        connection.setProcessGroup(processGroup);
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : dto.getBends()) {
            bendPoints.add(new Position(bend.getX(), bend.getY()));
        }
        connection.setBendPoints(bendPoints);
        final Long zIndex = dto.getzIndex();
        if (zIndex != null) {
            connection.setZIndex(zIndex);
        }
        if (dto.getLabelIndex() != null) {
            connection.setLabelIndex(dto.getLabelIndex());
        }
        List<FlowFilePrioritizer> newPrioritizers = null;
        final List<String> prioritizers = dto.getPrioritizers();
        if (prioritizers != null) {
            final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
            newPrioritizers = new ArrayList<>();
            for (final String className : newPrioritizersClasses) {
                try {
                    newPrioritizers.add(controller.createPrioritizer(className));
                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                    throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                }
            }
        }
        if (newPrioritizers != null) {
            connection.getFlowFileQueue().setPriorities(newPrioritizers);
        }
        if (dto.getBackPressureObjectThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold());
        }
        if (dto.getBackPressureDataSizeThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold());
        }
        if (dto.getFlowFileExpiration() != null) {
            connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration());
        }
        processGroup.addConnection(connection);
    }
    final List<Element> templateNodeList = getChildrenByTagName(processGroupElement, "template");
    for (final Element templateNode : templateNodeList) {
        final TemplateDTO templateDTO = TemplateUtils.parseDto(templateNode);
        final Template template = new Template(templateDTO);
        processGroup.addTemplate(template);
    }
    return processGroup;
}

Example 13 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class TemplatesRestController method getUnregisteredTemplates.

/**
 * This will populate the select drop down when a user asks to register a new template
 */
@GET
@Path("/unregistered")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the list of unregistered templates.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the templates.", response = TemplateDtoWrapper.class, responseContainer = "Set"), @ApiResponse(code = 500, message = "NiFi is unavailable.", response = RestResponseStatus.class) })
public Response getUnregisteredTemplates(@QueryParam("includeDetails") boolean includeDetails) {
    Set<TemplateDTO> nifiTemplates = nifiRestClient.getTemplates(includeDetails);
    // List<RegisteredTemplate> registeredTemplates = metadataService.getRegisteredTemplates();
    Set<TemplateDtoWrapper> dtos = new HashSet<>();
    for (final TemplateDTO dto : nifiTemplates) {
        RegisteredTemplate match = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByNiFiTemplateProperties(dto.getId(), dto.getName()));
        if (match == null) {
            dtos.add(new TemplateDtoWrapper(dto));
        }
    }
    return Response.ok(dtos).build();
}

Example 14 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getNiFiTemplateFlowProcessors.

/**
 * For a given Template and its related connection info to the reusable templates, walk the graph to return the Processors.
 * The system will first walk the incoming templateid.  If the {@code connectionInfo} parameter is set it will make the connections to the incoming template and continue walking those processors
 *
 * @param nifiTemplateId the NiFi templateId required to start walking the flow
 * @param connectionInfo the connections required to connect
 * @return a list of all the processors for a template and possible connections
 */
public List<RegisteredTemplate.FlowProcessor> getNiFiTemplateFlowProcessors(String nifiTemplateId, List<ReusableTemplateConnectionInfo> connectionInfo) {
    TemplateDTO templateDTO = nifiRestClient.getTemplateById(nifiTemplateId);
    // make the connection
    if (connectionInfo != null && !connectionInfo.isEmpty()) {
        Set<PortDTO> templatePorts = templateDTO.getSnippet().getOutputPorts();
        Map<String, PortDTO> outputPorts = templateDTO.getSnippet().getOutputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        Map<String, PortDTO> inputPorts = getReusableFeedInputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        connectionInfo.stream().forEach(reusableTemplateConnectionInfo -> {
            PortDTO outputPort = outputPorts.get(reusableTemplateConnectionInfo.getFeedOutputPortName());
            PortDTO inputPort = inputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName());
            ConnectionDTO connectionDTO = new ConnectionDTO();
            ConnectableDTO source = new ConnectableDTO();
            source.setName(reusableTemplateConnectionInfo.getFeedOutputPortName());
            source.setType(outputPort.getType());
            source.setId(outputPort.getId());
            source.setGroupId(outputPort.getParentGroupId());
            ConnectableDTO dest = new ConnectableDTO();
            dest.setName(inputPort.getName());
            dest.setType(inputPort.getType());
            dest.setId(inputPort.getId());
            dest.setGroupId(inputPort.getParentGroupId());
            connectionDTO.setSource(source);
            connectionDTO.setDestination(dest);
            connectionDTO.setId(UUID.randomUUID().toString());
            templateDTO.getSnippet().getConnections().add(connectionDTO);
        });
    }
    NifiFlowProcessGroup template = nifiRestClient.getTemplateFeedFlow(templateDTO);
    return template.getProcessorMap().values().stream().map(flowProcessor -> {
        RegisteredTemplate.FlowProcessor p = new RegisteredTemplate.FlowProcessor(flowProcessor.getId());
        p.setGroupId(flowProcessor.getParentGroupId());
        p.setType(flowProcessor.getType());
        p.setName(flowProcessor.getName());
        p.setFlowId(flowProcessor.getFlowId());
        p.setIsLeaf(flowProcessor.isLeaf());
        return p;
    }).collect(Collectors.toList());
}

Example 15 with TemplateDTO

use of org.apache.nifi.web.api.dto.TemplateDTO in project kylo by Teradata.

the class NiFiTemplateCache method geTemplate.

/**
 * Gets the populated Template with flow information first looking at the cache and then getting it from NiFi if it is stale
 *
 * @param nifiTemplateId the nifi template id
 * @param templateName   the name of the template
 * @return the populated template
 */
public TemplateDTO geTemplate(String nifiTemplateId, String templateName) {
    TemplateDTO templateDTO = null;
    TemplateDTO nifiTemplate = null;
    if (StringUtils.isNotBlank(nifiTemplateId)) {
        templateDTO = templateByIdCache.getIfPresent(nifiTemplateId);
        nifiTemplate = findSummaryById(nifiTemplateId);
    }
    if (templateDTO == null && StringUtils.isNotBlank(templateName)) {
        templateDTO = templateByNameCache.getIfPresent(templateName);
    }
    if (nifiTemplate == null && StringUtils.isNotBlank(templateName)) {
        nifiTemplate = findSummaryByName(templateName);
    }
    if (nifiTemplate != null) {
        if (StringUtils.isBlank(templateName)) {
            templateName = nifiTemplate.getName();
        }
        if (needsUpdate(nifiTemplate, templateDTO)) {
            log.info("Fetching NiFi template from NiFi {}, {}", nifiTemplateId, templateName);
            templateDTO = getPopulatedTemplate(nifiTemplateId, templateName);
            if (templateDTO != null) {
                log.info("Caching NiFi template {}, {}", nifiTemplateId, templateName);
                if (StringUtils.isNotBlank(nifiTemplateId)) {
                    templateByIdCache.put(nifiTemplateId, templateDTO);
                }
                if (StringUtils.isNotBlank(templateName)) {
                    templateByNameCache.put(templateName, templateDTO);
                }
            }
        } else {
            log.info("Returning Cached NiFi template {}, {}", nifiTemplateId, templateName);
        }
    }
    return templateDTO;
}