Examples with ControllerServiceDTO org.apache.nifi.web.api.dto.ControllerServiceDTO used on opensource projects

Example 21 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project nifi by apache.

the class ControllerServiceLoader method configureControllerService.

private static void configureControllerService(final ControllerServiceNode node, final Element controllerServiceElement, final StringEncryptor encryptor) {
    final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor);
    node.setAnnotationData(dto.getAnnotationData());
    node.setProperties(dto.getProperties());
}

Example 22 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project nifi by apache.

the class FingerprintFactory method addFlowControllerFingerprint.

private StringBuilder addFlowControllerFingerprint(final StringBuilder builder, final Element flowControllerElem, final FlowController controller) {
    // registries
    final Element registriesElement = DomUtils.getChild(flowControllerElem, "registries");
    if (registriesElement == null) {
        builder.append("NO_VALUE");
    } else {
        final List<Element> flowRegistryElems = DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry");
        if (flowRegistryElems.isEmpty()) {
            builder.append("NO_VALUE");
        } else {
            for (final Element flowRegistryElement : flowRegistryElems) {
                addFlowRegistryFingerprint(builder, flowRegistryElement);
            }
        }
    }
    // root group
    final Element rootGroupElem = (Element) DomUtils.getChildNodesByTagName(flowControllerElem, "rootGroup").item(0);
    addProcessGroupFingerprint(builder, rootGroupElem, controller);
    final Element controllerServicesElem = DomUtils.getChild(flowControllerElem, "controllerServices");
    if (controllerServicesElem != null) {
        final List<ControllerServiceDTO> serviceDtos = new ArrayList<>();
        for (final Element serviceElem : DomUtils.getChildElementsByTagName(controllerServicesElem, "controllerService")) {
            final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(serviceElem, encryptor);
            serviceDtos.add(dto);
        }
        Collections.sort(serviceDtos, new Comparator<ControllerServiceDTO>() {

            @Override
            public int compare(final ControllerServiceDTO o1, final ControllerServiceDTO o2) {
                if (o1 == null && o2 == null) {
                    return 0;
                }
                if (o1 == null && o2 != null) {
                    return 1;
                }
                if (o1 != null && o2 == null) {
                    return -1;
                }
                return o1.getId().compareTo(o2.getId());
            }
        });
        for (final ControllerServiceDTO dto : serviceDtos) {
            addControllerServiceFingerprint(builder, dto);
        }
    }
    final Element reportingTasksElem = DomUtils.getChild(flowControllerElem, "reportingTasks");
    if (reportingTasksElem != null) {
        final List<ReportingTaskDTO> reportingTaskDtos = new ArrayList<>();
        for (final Element taskElem : DomUtils.getChildElementsByTagName(reportingTasksElem, "reportingTask")) {
            final ReportingTaskDTO dto = FlowFromDOMFactory.getReportingTask(taskElem, encryptor);
            reportingTaskDtos.add(dto);
        }
        Collections.sort(reportingTaskDtos, new Comparator<ReportingTaskDTO>() {

            @Override
            public int compare(final ReportingTaskDTO o1, final ReportingTaskDTO o2) {
                if (o1 == null && o2 == null) {
                    return 0;
                }
                if (o1 == null && o2 != null) {
                    return 1;
                }
                if (o1 != null && o2 == null) {
                    return -1;
                }
                return o1.getId().compareTo(o2.getId());
            }
        });
        for (final ReportingTaskDTO dto : reportingTaskDtos) {
            addReportingTaskFingerprint(builder, dto);
        }
    }
    return builder;
}

Example 23 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 24 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class DatasourceModelTransform method updateDomain.

/**
 * Updates the specified domain object with properties from the specified REST object.
 *
 * @param domain the domain object
 * @param ds     the REST object
 */
private void updateDomain(@Nonnull final com.thinkbiganalytics.metadata.api.datasource.UserDatasource domain, @Nonnull final JdbcDatasource ds) {
    updateDomain(domain, (UserDatasource) ds);
    domain.getDetails().map(JdbcDatasourceDetails.class::cast).ifPresent(details -> {
        // Look for changed properties
        final Map<String, String> properties = new HashMap<>();
        if (StringUtils.isNotBlank(ds.getDatabaseConnectionUrl())) {
            properties.put(DatasourceConstants.DATABASE_CONNECTION_URL, ds.getDatabaseConnectionUrl());
        }
        if (StringUtils.isNotBlank(ds.getDatabaseDriverClassName())) {
            properties.put(DatasourceConstants.DATABASE_DRIVER_CLASS_NAME, ds.getDatabaseDriverClassName());
        }
        if (ds.getDatabaseDriverLocation() != null) {
            properties.put(DatasourceConstants.DATABASE_DRIVER_LOCATION, ds.getDatabaseDriverLocation());
        }
        if (ds.getDatabaseUser() != null) {
            properties.put(DatasourceConstants.DATABASE_USER, ds.getDatabaseUser());
        }
        if (ds.getPassword() != null) {
            details.setPassword(encryptor.encrypt(ds.getPassword()));
            properties.put(DatasourceConstants.PASSWORD, StringUtils.isNotEmpty(ds.getPassword()) ? ds.getPassword() : null);
        }
        // Update or create the controller service
        ControllerServiceDTO controllerService = null;
        if (details.getControllerServiceId().isPresent()) {
            controllerService = new ControllerServiceDTO();
            controllerService.setId(details.getControllerServiceId().get());
            controllerService.setName(ds.getName());
            controllerService.setComments(ds.getDescription());
            controllerService.setProperties(properties);
            try {
                controllerService = nifiRestClient.controllerServices().updateServiceAndReferencingComponents(controllerService);
                ds.setControllerServiceId(controllerService.getId());
            } catch (final NifiComponentNotFoundException e) {
                log.warn("Controller service is missing for datasource: {}", domain.getId(), e);
                controllerService = null;
            }
        }
        if (controllerService == null) {
            controllerService = new ControllerServiceDTO();
            controllerService.setType("org.apache.nifi.dbcp.DBCPConnectionPool");
            controllerService.setName(ds.getName());
            controllerService.setComments(ds.getDescription());
            controllerService.setProperties(properties);
            final ControllerServiceDTO newControllerService = nifiRestClient.controllerServices().create(controllerService);
            try {
                nifiRestClient.controllerServices().updateStateById(newControllerService.getId(), NiFiControllerServicesRestClient.State.ENABLED);
            } catch (final NifiClientRuntimeException nifiException) {
                log.error("Failed to enable controller service for datasource: {}", domain.getId(), nifiException);
                nifiRestClient.controllerServices().disableAndDeleteAsync(newControllerService.getId());
                throw nifiException;
            }
            details.setControllerServiceId(newControllerService.getId());
            ds.setControllerServiceId(newControllerService.getId());
        }
    });
}

Example 25 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class DerivedDatasourceFactory method parseDataTransformControllerServiceProperties.

private Map<String, String> parseDataTransformControllerServiceProperties(DatasourceDefinition datasourceDefinition, String controllerServiceName) {
    Map<String, String> properties = new HashMap<>();
    if (datasourceDefinition != null) {
        try {
            if (StringUtils.isNotBlank(controllerServiceName)) {
                // {Source Database Connection:Database Connection URL}
                List<String> controllerServiceProperties = datasourceDefinition.getDatasourcePropertyKeys().stream().filter(k -> k.matches("\\{" + JDBC_CONNECTION_KEY + ":(.*)\\}")).collect(Collectors.toList());
                List<String> serviceProperties = new ArrayList<>();
                controllerServiceProperties.stream().forEach(p -> {
                    String property = p.substring(StringUtils.indexOf(p, ":") + 1, p.length() - 1);
                    serviceProperties.add(property);
                });
                ControllerServiceDTO csDto = nifiControllerServiceProperties.getControllerServiceByName(controllerServiceName);
                if (csDto != null) {
                    serviceProperties.stream().forEach(p -> {
                        if (csDto != null) {
                            String value = csDto.getProperties().get(p);
                            if (value != null) {
                                properties.put(p, value);
                            }
                        }
                    });
                }
            }
        } catch (Exception e) {
            log.warn("An error occurred trying to parse controller service properties for data transformation when deriving the datasource for {}, {}. {} ", datasourceDefinition.getDatasourceType(), datasourceDefinition.getConnectionType(), e.getMessage(), e);
        }
    }
    return properties;
}