Examples with ReusableTemplateConnectionInfo com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo used on opensource projects

Example 1 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class ImportReusableTemplate method validateOutputPortConnections.

private boolean validateOutputPortConnections(NifiProcessGroup newTemplateInstance) {
    // Validate port connections
    newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().forEach(portDTO -> {
        if (portDTO.getValidationErrors() != null && !portDTO.getValidationErrors().isEmpty()) {
            importTemplate.setReusableFlowOutputPortConnectionsNeeded(true);
        }
        ReusableTemplateConnectionInfo connectionInfo = new ReusableTemplateConnectionInfo();
        connectionInfo.setFeedOutputPortName(portDTO.getName());
        // attempt to prefill it with the previous connection if it existed
        ConnectionDTO reusableTemplateInputPortConnection = findReusableTemplateInputPortConnectionForOutputPort(portDTO);
        if (reusableTemplateInputPortConnection != null) {
            connectionInfo.setInputPortDisplayName(reusableTemplateInputPortConnection.getSource().getName());
            connectionInfo.setReusableTemplateInputPortName(reusableTemplateInputPortConnection.getSource().getName());
            String processGroupName = findReusableTemplateProcessGroup(reusableTemplateInputPortConnection.getDestination().getGroupId()).map(processGroupDTO -> processGroupDTO.getName()).orElse(null);
            connectionInfo.setReusableTemplateProcessGroupName(processGroupName);
        }
        importTemplate.addReusableTemplateConnection(connectionInfo);
    });
    return importTemplate.isSuccess() && importTemplate.isValid() && !importTemplate.isReusableFlowOutputPortConnectionsNeeded();
}

Example 2 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultFeedManagerFeedService method deployFeed.

private NifiFeed deployFeed(final FeedMetadata feedMetadata, com.thinkbiganalytics.metadata.api.versioning.EntityVersion<Feed.ID, Feed> version) throws DeployFeedException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    boolean enabled = false;
    if (feedMetadata.isActive()) {
        feedMetadata.setState(Feed.State.ENABLED.name());
        enabled = true;
    } else {
        feedMetadata.setState(Feed.State.DISABLED.name());
    }
    // Store ref to the originalFeedProperties before resolving and merging with the template
    List<NifiProperty> originalFeedProperties = feedMetadata.getProperties();
    // Get all the properties for the metadata
    RegisteredTemplate registeredTemplate = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).templateName(feedMetadata.getTemplateName()).isFeedEdit(true).includeSensitiveProperties(true).build());
    // Copy the registered template properties it a new list so it doest get updated
    List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
    // Update the template properties with the feedMetadata properties
    NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PropertyUpdateMode.UPDATE_ALL_PROPERTIES);
    registeredTemplate.setProperties(templateProperties);
    feedMetadata.setProperties(registeredTemplate.getProperties());
    feedMetadata.setRegisteredTemplate(registeredTemplate);
    // Skip any properties that the user supplied which are not ${ values
    List<NifiProperty> propertiesToSkip = originalFeedProperties.stream().filter(property -> !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    List<NifiProperty> templatePropertiesToSkip = registeredTemplate.getProperties().stream().filter(property -> property.isSelected() && !propertyExpressionResolver.containsVariablesPatterns(property.getValue())).collect(Collectors.toList());
    if (templatePropertiesToSkip != null && !templatePropertiesToSkip.isEmpty()) {
        propertiesToSkip.addAll(templatePropertiesToSkip);
    }
    // Resolve any ${metadata.} properties
    propertyExpressionResolver.resolvePropertyExpressions(feedMetadata, propertiesToSkip);
    // decrypt the metadata
    feedModelTransform.decryptSensitivePropertyValues(feedMetadata);
    // if this is the very first version we need to enable it later (after the data has been sync'd with ops manager)
    boolean enableLater = false;
    if (enabled && version.isFirstVersion()) {
        enabled = false;
        enableLater = true;
        feedMetadata.setState(FeedMetadata.STATE.DISABLED.name());
    }
    CreateFeedBuilder feedBuilder = CreateFeedBuilder.newFeed(nifiRestClient, nifiFlowCache, feedMetadata, registeredTemplate.getNifiTemplateId(), propertyExpressionResolver, propertyDescriptorTransform, niFiObjectCache, templateConnectionUtil).enabled(enabled).setOriginalFeedProperties(originalFeedProperties).removeInactiveVersionedProcessGroup(removeInactiveNifiVersionedFeedFlows).autoAlign(nifiAutoFeedsAlignAfterSave).withNiFiTemplateCache(niFiTemplateCache);
    if (registeredTemplate.isReusableTemplate()) {
        feedBuilder.setReusableTemplate(true);
        feedMetadata.setIsReusableFeed(true);
    } else {
        feedBuilder.inputProcessorType(feedMetadata.getInputProcessorType()).feedSchedule(feedMetadata.getSchedule()).properties(feedMetadata.getProperties());
        if (registeredTemplate.usesReusableTemplate()) {
            for (ReusableTemplateConnectionInfo connection : registeredTemplate.getReusableTemplateConnections()) {
                feedBuilder.addInputOutputPort(new InputOutputPort(connection.getReusableTemplateInputPortName(), connection.getFeedOutputPortName()));
            }
        }
    }
    stopwatch.stop();
    log.debug("Time to prepare data for saving feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    stopwatch.start();
    NifiProcessGroup entity = feedBuilder.build();
    stopwatch.stop();
    log.debug("Time to save feed in NiFi: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    NifiFeed feed = new NifiFeed(feedMetadata, entity);
    // Set the original feedProperties back to the feed
    feedMetadata.setProperties(originalFeedProperties);
    // Encrypt the metadata properties
    feedModelTransform.encryptSensitivePropertyValues(feedMetadata);
    if (entity.isSuccess()) {
        feedMetadata.setNifiProcessGroupId(entity.getProcessGroupEntity().getId());
        try {
            stopwatch.start();
            saveDeployedFeed(feedMetadata, version);
            // tell NiFi if this is a streaming feed or not
            if (feedMetadata.getRegisteredTemplate().isStream()) {
                streamingFeedJmsNotificationService.updateNiFiStatusJMSTopic(entity, feedMetadata);
            }
            feed.setSuccess(true);
            feed.setEnableAfterSave(enableLater);
            stopwatch.stop();
            log.debug("Time to saveFeed in Kylo: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            stopwatch.start();
            feedBuilder.checkAndRemoveVersionedProcessGroup();
        } catch (Exception e) {
            feed.setSuccess(false);
            feed.addErrorMessage(e);
        }
    } else {
        feed.setSuccess(false);
    }
    if (!feed.isSuccess()) {
        if (!entity.isRolledBack()) {
            try {
                feedBuilder.rollback();
            } catch (FeedRollbackException rollbackException) {
                log.error("Error rolling back feed {}. {} ", feedMetadata.getCategoryAndFeedName(), rollbackException.getMessage());
                feed.addErrorMessage("Error occurred in rolling back the Feed.");
            }
            entity.setRolledBack(true);
        }
        throw new DeployFeedException(feed);
    }
    // Move to isSuccess block??
    feedHistoryDataReindexingService.updateHistoryDataReindexingFeedsAvailableCache(feedMetadata);
    return feed;
}

Example 3 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultTemplateExporter method gatherConnectedReusableTemplates.

private void gatherConnectedReusableTemplates(List<String> connectingReusableTemplates, Set<String> connectedTemplateIds, Set<ReusableTemplateConnectionInfo> connectingOutputPortConnectionMetadata, List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos, ProcessGroupFlowDTO reusableTemplateFlow) {
    for (ReusableTemplateConnectionInfo reusableTemplateConnectionInfo : reusableTemplateConnectionInfos) {
        String inputName = reusableTemplateConnectionInfo.getReusableTemplateInputPortName();
        // find the process group instance in the 'reusable_templates' group in NiFi for this input port
        Optional<ProcessGroupDTO> processGroupDTO = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getName().equals(inputName) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).flatMap(connectionDTO -> reusableTemplateFlow.getFlow().getProcessGroups().stream().map(processGroupEntity -> processGroupEntity.getComponent()).filter(groupDTO -> groupDTO.getId().equals(connectionDTO.getDestination().getGroupId()))).findFirst();
        if (processGroupDTO.isPresent()) {
            // walk the output ports to find any other connecting templates
            List<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new ArrayList<>();
            List<ConnectionDTO> outputPortConnections = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getGroupId().equals(processGroupDTO.get().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)).collect(Collectors.toList());
            for (ConnectionDTO outputConnection : outputPortConnections) {
                // walk these and get their templates
                // first get the connection metadata info needed
                // 1 find the reusable template input port for this connected template
                Optional<ConnectionDTO> inputPortToProcessGroupConnection = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getDestination().getId().equals(outputConnection.getDestination().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).findFirst();
                if (inputPortToProcessGroupConnection.isPresent()) {
                    ReusableTemplateConnectionInfo connectionInfo = new ReusableTemplateConnectionInfo();
                    connectionInfo.setFeedOutputPortName(outputConnection.getSource().getName());
                    connectionInfo.setReusableTemplateInputPortName(inputPortToProcessGroupConnection.get().getSource().getName());
                    connectionInfo.setInputPortDisplayName(inputPortToProcessGroupConnection.get().getSource().getName());
                    String processGroupName = reusableTemplateFlow.getFlow().getProcessGroups().stream().filter(processGroupEntity -> processGroupEntity.getComponent().getId().equals(inputPortToProcessGroupConnection.get().getDestination().getGroupId())).map(processGroupEntity -> processGroupEntity.getComponent().getName()).findFirst().orElse(null);
                    connectionInfo.setReusableTemplateProcessGroupName(processGroupName);
                    outputPortConnectionMetadata.add(connectionInfo);
                    // recursively walk these flows and gather other output port connections
                    connectingOutputPortConnectionMetadata.add(connectionInfo);
                }
            // also add in the process group if it doesnt connect
            }
            if (!outputPortConnectionMetadata.isEmpty()) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, connectingOutputPortConnectionMetadata, outputPortConnectionMetadata, reusableTemplateFlow);
            }
        }
        // find the template that has the input port name
        Map<String, String> map = nifiRestClient.getTemplatesAsXmlMatchingInputPortName(inputName, reusableTemplateConnectionInfo.getReusableTemplateProcessGroupName());
        if (map != null && !map.isEmpty()) {
            for (Map.Entry<String, String> entry : map.entrySet()) {
                String portTemplateId = entry.getKey();
                if (!connectedTemplateIds.contains(portTemplateId)) {
                    connectedTemplateIds.add(portTemplateId);
                    connectingReusableTemplates.add(entry.getValue());
                }
            }
        }
    }
}

Example 4 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getNiFiTemplateFlowProcessors.

/**
 * For a given Template and its related connection info to the reusable templates, walk the graph to return the Processors.
 * The system will first walk the incoming templateid.  If the {@code connectionInfo} parameter is set it will make the connections to the incoming template and continue walking those processors
 *
 * @param nifiTemplateId the NiFi templateId required to start walking the flow
 * @param connectionInfo the connections required to connect
 * @return a list of all the processors for a template and possible connections
 */
public List<RegisteredTemplate.FlowProcessor> getNiFiTemplateFlowProcessors(String nifiTemplateId, List<ReusableTemplateConnectionInfo> connectionInfo) {
    TemplateDTO templateDTO = nifiRestClient.getTemplateById(nifiTemplateId);
    // make the connection
    if (connectionInfo != null && !connectionInfo.isEmpty()) {
        Set<PortDTO> templatePorts = templateDTO.getSnippet().getOutputPorts();
        Map<String, PortDTO> outputPorts = templateDTO.getSnippet().getOutputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        Map<String, PortDTO> inputPorts = getReusableFeedInputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        connectionInfo.stream().forEach(reusableTemplateConnectionInfo -> {
            PortDTO outputPort = outputPorts.get(reusableTemplateConnectionInfo.getFeedOutputPortName());
            PortDTO inputPort = inputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName());
            ConnectionDTO connectionDTO = new ConnectionDTO();
            ConnectableDTO source = new ConnectableDTO();
            source.setName(reusableTemplateConnectionInfo.getFeedOutputPortName());
            source.setType(outputPort.getType());
            source.setId(outputPort.getId());
            source.setGroupId(outputPort.getParentGroupId());
            ConnectableDTO dest = new ConnectableDTO();
            dest.setName(inputPort.getName());
            dest.setType(inputPort.getType());
            dest.setId(inputPort.getId());
            dest.setGroupId(inputPort.getParentGroupId());
            connectionDTO.setSource(source);
            connectionDTO.setDestination(dest);
            connectionDTO.setId(UUID.randomUUID().toString());
            templateDTO.getSnippet().getConnections().add(connectionDTO);
        });
    }
    NifiFlowProcessGroup template = nifiRestClient.getTemplateFeedFlow(templateDTO);
    return template.getProcessorMap().values().stream().map(flowProcessor -> {
        RegisteredTemplate.FlowProcessor p = new RegisteredTemplate.FlowProcessor(flowProcessor.getId());
        p.setGroupId(flowProcessor.getParentGroupId());
        p.setType(flowProcessor.getType());
        p.setName(flowProcessor.getName());
        p.setFlowId(flowProcessor.getFlowId());
        p.setIsLeaf(flowProcessor.isLeaf());
        return p;
    }).collect(Collectors.toList());
}

Example 5 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class ImportConnectedReusableTemplateIT method registerConnectedReusableTemplate.

public ConnectedTemplate registerConnectedReusableTemplate() {
    URL resource = IntegrationTestBase.class.getResource("connecting_reusable_flow.xml");
    ImportTemplate template1 = importReusableFlowXmlTemplate(resource.getPath(), null);
    String template1ProcessGroupId = template1.getTemplateResults().getProcessGroupEntity().getId();
    // Now import a reusable flow that has additional output ports in it.
    // Kylo will prompt to connect this to another reusable flow
    URL resource2 = IntegrationTestBase.class.getResource("reusable-flow1.xml");
    ImportTemplate template2 = importReusableFlowXmlTemplate(resource2.getPath(), null);
    // verify it failed
    Assert.assertFalse(template2.isSuccess());
    Assert.assertTrue(template2.isReusableFlowOutputPortConnectionsNeeded());
    // reassign the connections to connect to template1
    ReusableTemplateConnectionInfo connectionInfo = template2.getReusableTemplateConnections().stream().filter(conn -> "to another flow".equalsIgnoreCase(conn.getFeedOutputPortName())).findFirst().orElse(null);
    // Obtain the reusable connection input ports
    // get v1/feedmgr/nifi/reusable-input-ports
    PortDTO[] reusableInputPorts = getReusableInputPorts();
    // the 'connecting_reusable_flow.xml' has an input port named 'from reusable port'.
    // find it and try to connect it to this one
    PortDTO connectingPort = Arrays.stream(reusableInputPorts).filter(portDTO -> portDTO.getName().equalsIgnoreCase("from reusable port")).findFirst().orElse(null);
    if (connectingPort != null) {
        connectionInfo.setInputPortDisplayName(connectingPort.getName());
        connectionInfo.setReusableTemplateInputPortName(connectingPort.getName());
        connectionInfo.setReusableTemplateProcessGroupName(template1.getTemplateResults().getProcessGroupEntity().getName());
    }
    template2 = importReusableFlowXmlTemplate(resource2.getPath(), connectionInfo);
    Assert.assertTrue(template2.isSuccess());
    Assert.assertFalse(template2.isReusableFlowOutputPortConnectionsNeeded());
    // get the flow for the parent processor and verify it is connected to the other reusable flow
    NifiFlowProcessGroup flow = getFlow(template2.getTemplateResults().getProcessGroupEntity().getId());
    Assert.assertNotNull(flow);
    boolean testUpdate2ProcessorExists = flow.getProcessorMap().values().stream().anyMatch(p -> "test-update2".equalsIgnoreCase(p.getName()));
    Assert.assertTrue(testUpdate2ProcessorExists);
    return new ConnectedTemplate(template1, template2);
}