Examples with ProcessGroupDTO org.apache.nifi.web.api.dto.ProcessGroupDTO used on opensource projects

Example 21 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project nifi by apache.

the class FlowController method verifyComponentTypesInSnippet.

public void verifyComponentTypesInSnippet(final FlowSnippetDTO templateContents) {
    final Map<String, Set<BundleCoordinate>> processorClasses = new HashMap<>();
    for (final Class<?> c : ExtensionManager.getExtensions(Processor.class)) {
        final String name = c.getName();
        processorClasses.put(name, ExtensionManager.getBundles(name).stream().map(bundle -> bundle.getBundleDetails().getCoordinate()).collect(Collectors.toSet()));
    }
    verifyProcessorsInSnippet(templateContents, processorClasses);
    final Map<String, Set<BundleCoordinate>> controllerServiceClasses = new HashMap<>();
    for (final Class<?> c : ExtensionManager.getExtensions(ControllerService.class)) {
        final String name = c.getName();
        controllerServiceClasses.put(name, ExtensionManager.getBundles(name).stream().map(bundle -> bundle.getBundleDetails().getCoordinate()).collect(Collectors.toSet()));
    }
    verifyControllerServicesInSnippet(templateContents, controllerServiceClasses);
    final Set<String> prioritizerClasses = new HashSet<>();
    for (final Class<?> c : ExtensionManager.getExtensions(FlowFilePrioritizer.class)) {
        prioritizerClasses.add(c.getName());
    }
    final Set<ConnectionDTO> allConns = new HashSet<>();
    allConns.addAll(templateContents.getConnections());
    for (final ProcessGroupDTO childGroup : templateContents.getProcessGroups()) {
        allConns.addAll(findAllConnections(childGroup));
    }
    for (final ConnectionDTO conn : allConns) {
        final List<String> prioritizers = conn.getPrioritizers();
        if (prioritizers != null) {
            for (final String prioritizer : prioritizers) {
                if (!prioritizerClasses.contains(prioritizer)) {
                    throw new IllegalStateException("Invalid FlowFile Prioritizer Type: " + prioritizer);
                }
            }
        }
    }
}

Example 22 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 23 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(String feedGroupId, String feedCategoryId) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    for (InputOutputPort port : inputOutputPorts) {
        stopwatch.start();
        restClient.connectFeedToGlobalTemplate(feedGroupId, port.getOutputPortName(), feedCategoryId, reusableTemplateCategoryGroupId, port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 24 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project nifi-minifi by apache.

the class FlowSnippetDTOEnricher method enrich.

public void enrich(FlowSnippetDTO flowSnippetDTO, final String encodingVersion) {
    List<FlowSnippetDTO> allFlowSnippets = getAllFlowSnippets(flowSnippetDTO);
    Set<RemoteProcessGroupDTO> remoteProcessGroups = getAll(allFlowSnippets, FlowSnippetDTO::getRemoteProcessGroups).collect(Collectors.toSet());
    Map<String, String> connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName));
    Map<String, String> rpgIdToTargetIdMap = new HashMap<>();
    for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) {
        final RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents();
        final Set<RemoteProcessGroupPortDTO> rpgInputPortDtos = nullToEmpty(contents.getInputPorts());
        final Set<RemoteProcessGroupPortDTO> rpgOutputPortDtos = nullToEmpty(contents.getOutputPorts());
        switch(encodingVersion) {
            case "1.2":
                // Map all port DTOs to their respective targetIds
                rpgIdToTargetIdMap.putAll(Stream.concat(rpgInputPortDtos.stream(), rpgOutputPortDtos.stream()).collect(Collectors.toMap(RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getTargetId)));
                break;
            default:
                break;
        }
        addConnectables(connectableNameMap, rpgInputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
        addConnectables(connectableNameMap, rpgOutputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId);
    }
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getInputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getOutputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName);
    final Set<ConnectionDTO> connections = getAll(allFlowSnippets, FlowSnippetDTO::getConnections).collect(Collectors.toSet());
    // Enrich connection endpoints using known names and overriding with targetIds for remote ports
    for (ConnectionDTO connection : connections) {
        setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap);
        setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap);
    }
    // Override any ids that are for Remote Ports to use their target Ids where available
    connections.stream().flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())).filter(connectable -> connectable.getType().equals(ConnectableType.REMOTE_OUTPUT_PORT.toString()) || connectable.getType().equals(ConnectableType.REMOTE_INPUT_PORT.toString())).forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId())));
    // Establish unique names for connections
    for (ConnectionDTO connection : connections) {
        if (StringUtil.isNullOrEmpty(connection.getName())) {
            StringBuilder name = new StringBuilder();
            ConnectableDTO connectionSource = connection.getSource();
            name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap));
            name.append("/");
            if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) {
                name.append(connection.getSelectedRelationships().iterator().next());
            }
            name.append("/");
            ConnectableDTO connectionDestination = connection.getDestination();
            name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap));
            connection.setName(name.toString());
        }
    }
    nullToEmpty(flowSnippetDTO.getProcessGroups()).stream().map(ProcessGroupDTO::getContents).forEach(snippetDTO -> enrich(snippetDTO, encodingVersion));
}

Example 25 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class FeedManagerMetadataService method deleteFeed.

@Override
public void deleteFeed(@Nonnull final String feedId) {
    // First check if this should be allowed.
    accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ADMIN_FEEDS);
    feedProvider.checkFeedPermission(feedId, FeedAccessControl.DELETE);
    // Step 1: Fetch feed metadata
    final FeedMetadata feed = feedProvider.getFeedById(feedId);
    if (feed == null) {
        throw new IllegalArgumentException("Unknown feed: " + feedId);
    }
    // Step 2: Check category permissions
    categoryProvider.checkCategoryPermission(feed.getCategoryId(), CategoryAccessControl.CREATE_FEED);
    // Step 3: Check for dependent feeds
    if (feed.getUsedByFeeds() != null && !feed.getUsedByFeeds().isEmpty()) {
        final List<String> systemNames = feed.getUsedByFeeds().stream().map(FeedSummary::getCategoryAndFeedSystemName).collect(Collectors.toList());
        throw new IllegalStateException("Feed is referenced by " + feed.getUsedByFeeds().size() + " other feeds: " + systemNames);
    }
    // check SLAs
    metadataAccess.read(() -> {
        boolean hasSlas = serviceLevelAgreementService.hasServiceLevelAgreements(feedProvider.resolveFeed(feedId));
        if (hasSlas) {
            log.error("Unable to delete " + feed.getCategoryAndFeedDisplayName() + ".  1 or more SLAs exist for this feed. ");
            throw new IllegalStateException("Unable to delete the feed. 1 or more Service Level agreements exist for this feed " + feed.getCategoryAndFeedDisplayName() + ".  Please delete the SLA's, or remove the feed from the SLA's and try again.");
        }
    }, MetadataAccess.SERVICE);
    // Step 4: Delete hadoop authorization security policies if they exists
    if (hadoopAuthorizationService != null) {
        metadataAccess.read(() -> {
            Feed domainFeed = feedModelTransform.feedToDomain(feed);
            String hdfsPaths = (String) domainFeed.getProperties().get(HadoopAuthorizationService.REGISTRATION_HDFS_FOLDERS);
            hadoopAuthorizationService.deleteHivePolicy(feed.getSystemCategoryName(), feed.getSystemFeedName());
            hadoopAuthorizationService.deleteHdfsPolicy(feed.getSystemCategoryName(), feed.getSystemFeedName(), HadoopAuthorizationService.convertNewlineDelimetedTextToList(hdfsPaths));
        });
    }
    // Step 5: Enable NiFi cleanup flow
    boolean hasCleanupFlow = false;
    final ProcessGroupDTO feedProcessGroup;
    final ProcessGroupDTO categoryProcessGroup = nifiRestClient.getProcessGroupByName("root", feed.getSystemCategoryName(), false, true);
    if (categoryProcessGroup != null) {
        feedProcessGroup = NifiProcessUtil.findFirstProcessGroupByName(categoryProcessGroup.getContents().getProcessGroups(), feed.getSystemFeedName());
        if (feedProcessGroup != null) {
            hasCleanupFlow = nifiRestClient.setInputAsRunningByProcessorMatchingType(feedProcessGroup.getId(), "com.thinkbiganalytics.nifi.v2.metadata.TriggerCleanup");
        }
    }
    if (hasCleanupFlow) {
        // Wait for input processor to start
        try {
            Thread.sleep(cleanupDelay);
        } catch (InterruptedException e) {
        // ignored
        }
    }
    // Step 6: Signal the cleanup event.
    notifyFeedCleanup(feed);
    // Step 7: Optionally wait for the cleanup job to finish if this feed has a cleanup flow.
    if (hasCleanupFlow) {
        waitForFeedCleanup(feed);
    }
    // Step 8: Remove feed from NiFi
    if (categoryProcessGroup != null) {
        final Set<ConnectionDTO> connections = categoryProcessGroup.getContents().getConnections();
        for (ProcessGroupDTO processGroup : NifiProcessUtil.findProcessGroupsByFeedName(categoryProcessGroup.getContents().getProcessGroups(), feed.getSystemFeedName())) {
            nifiRestClient.deleteProcessGroupAndConnections(processGroup, connections);
        }
        // disable any ports that are not connected to anything
        nifiRestClient.disableDisconnectedPorts(categoryProcessGroup);
    }
    // Step 9: Delete database entries
    feedProvider.deleteFeed(feedId);
}