Examples with NifiFlowProcessGroup com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup used on opensource projects

Example 1 with NifiFlowProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup in project kylo by Teradata.

the class NifiIntegrationRestController method getFlow.

@GET
@Path(FLOW + "/{processGroupId}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the flow of the specified process group.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the flow.", response = NifiFlowProcessGroup.class), @ApiResponse(code = 500, message = "The process group is unavailable.", response = RestResponseStatus.class) })
public Response getFlow(@PathParam("processGroupId") String processGroupId) {
    accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ADMIN_FEEDS);
    NifiFlowProcessGroup flow = legacyNifiRestClient.getFeedFlow(processGroupId);
    NifiFlowDeserializer.prepareForSerialization(flow);
    return Response.ok(flow).build();
}

Example 2 with NifiFlowProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup in project kylo by Teradata.

the class DefaultNiFiFlowVisitorClient method getTemplateFeedFlow.

public NifiFlowProcessGroup getTemplateFeedFlow(TemplateDTO template) {
    ProcessGroupDTO parentProcessGroup = new ProcessGroupDTO();
    parentProcessGroup.setId(UUID.randomUUID().toString());
    parentProcessGroup.setParentGroupId(UUID.randomUUID().toString());
    parentProcessGroup.setName(template.getName());
    parentProcessGroup.setContents(template.getSnippet());
    NifiConnectionOrderVisitorCache cache = new NifiConnectionOrderVisitorCache();
    Collection<ProcessGroupDTO> groups = NifiProcessUtil.getProcessGroups(parentProcessGroup);
    cache.add(parentProcessGroup);
    // add the snippet as its own process group
    if (template.getSnippet().getProcessors() != null) {
        // find the first processor and get its parent group id
        Optional<ProcessorDTO> firstProcessor = template.getSnippet().getProcessors().stream().findFirst();
        if (firstProcessor.isPresent()) {
            String groupId = firstProcessor.get().getParentGroupId();
            ProcessGroupDTO snippetGroup = new ProcessGroupDTO();
            snippetGroup.setId(groupId);
            snippetGroup.setParentGroupId(template.getGroupId());
            snippetGroup.setContents(template.getSnippet());
            cache.add(snippetGroup);
        }
    }
    if (groups != null) {
        groups.stream().forEach(group -> cache.add(group));
    }
    // add any remote ProcessGroups
    if (template.getSnippet().getRemoteProcessGroups() != null) {
        template.getSnippet().getRemoteProcessGroups().stream().forEach(remoteProcessGroupDTO -> cache.add(remoteProcessGroupDTO));
    }
    NifiVisitableProcessGroup visitableGroup = getFlowOrder(parentProcessGroup, cache, false);
    NifiFlowProcessGroup flow = new NifiFlowBuilder().build(visitableGroup);
    return flow;
}

Example 3 with NifiFlowProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getNiFiTemplateFlowProcessors.

/**
 * For a given Template and its related connection info to the reusable templates, walk the graph to return the Processors.
 * The system will first walk the incoming templateid.  If the {@code connectionInfo} parameter is set it will make the connections to the incoming template and continue walking those processors
 *
 * @param nifiTemplateId the NiFi templateId required to start walking the flow
 * @param connectionInfo the connections required to connect
 * @return a list of all the processors for a template and possible connections
 */
public List<RegisteredTemplate.FlowProcessor> getNiFiTemplateFlowProcessors(String nifiTemplateId, List<ReusableTemplateConnectionInfo> connectionInfo) {
    TemplateDTO templateDTO = nifiRestClient.getTemplateById(nifiTemplateId);
    // make the connection
    if (connectionInfo != null && !connectionInfo.isEmpty()) {
        Set<PortDTO> templatePorts = templateDTO.getSnippet().getOutputPorts();
        Map<String, PortDTO> outputPorts = templateDTO.getSnippet().getOutputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        Map<String, PortDTO> inputPorts = getReusableFeedInputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        connectionInfo.stream().forEach(reusableTemplateConnectionInfo -> {
            PortDTO outputPort = outputPorts.get(reusableTemplateConnectionInfo.getFeedOutputPortName());
            PortDTO inputPort = inputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName());
            ConnectionDTO connectionDTO = new ConnectionDTO();
            ConnectableDTO source = new ConnectableDTO();
            source.setName(reusableTemplateConnectionInfo.getFeedOutputPortName());
            source.setType(outputPort.getType());
            source.setId(outputPort.getId());
            source.setGroupId(outputPort.getParentGroupId());
            ConnectableDTO dest = new ConnectableDTO();
            dest.setName(inputPort.getName());
            dest.setType(inputPort.getType());
            dest.setId(inputPort.getId());
            dest.setGroupId(inputPort.getParentGroupId());
            connectionDTO.setSource(source);
            connectionDTO.setDestination(dest);
            connectionDTO.setId(UUID.randomUUID().toString());
            templateDTO.getSnippet().getConnections().add(connectionDTO);
        });
    }
    NifiFlowProcessGroup template = nifiRestClient.getTemplateFeedFlow(templateDTO);
    return template.getProcessorMap().values().stream().map(flowProcessor -> {
        RegisteredTemplate.FlowProcessor p = new RegisteredTemplate.FlowProcessor(flowProcessor.getId());
        p.setGroupId(flowProcessor.getParentGroupId());
        p.setType(flowProcessor.getType());
        p.setName(flowProcessor.getName());
        p.setFlowId(flowProcessor.getFlowId());
        p.setIsLeaf(flowProcessor.isLeaf());
        return p;
    }).collect(Collectors.toList());
}

Example 4 with NifiFlowProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup in project kylo by Teradata.

the class ImportConnectedReusableTemplateIT method registerConnectedReusableTemplate.

public ConnectedTemplate registerConnectedReusableTemplate() {
    URL resource = IntegrationTestBase.class.getResource("connecting_reusable_flow.xml");
    ImportTemplate template1 = importReusableFlowXmlTemplate(resource.getPath(), null);
    String template1ProcessGroupId = template1.getTemplateResults().getProcessGroupEntity().getId();
    // Now import a reusable flow that has additional output ports in it.
    // Kylo will prompt to connect this to another reusable flow
    URL resource2 = IntegrationTestBase.class.getResource("reusable-flow1.xml");
    ImportTemplate template2 = importReusableFlowXmlTemplate(resource2.getPath(), null);
    // verify it failed
    Assert.assertFalse(template2.isSuccess());
    Assert.assertTrue(template2.isReusableFlowOutputPortConnectionsNeeded());
    // reassign the connections to connect to template1
    ReusableTemplateConnectionInfo connectionInfo = template2.getReusableTemplateConnections().stream().filter(conn -> "to another flow".equalsIgnoreCase(conn.getFeedOutputPortName())).findFirst().orElse(null);
    // Obtain the reusable connection input ports
    // get v1/feedmgr/nifi/reusable-input-ports
    PortDTO[] reusableInputPorts = getReusableInputPorts();
    // the 'connecting_reusable_flow.xml' has an input port named 'from reusable port'.
    // find it and try to connect it to this one
    PortDTO connectingPort = Arrays.stream(reusableInputPorts).filter(portDTO -> portDTO.getName().equalsIgnoreCase("from reusable port")).findFirst().orElse(null);
    if (connectingPort != null) {
        connectionInfo.setInputPortDisplayName(connectingPort.getName());
        connectionInfo.setReusableTemplateInputPortName(connectingPort.getName());
        connectionInfo.setReusableTemplateProcessGroupName(template1.getTemplateResults().getProcessGroupEntity().getName());
    }
    template2 = importReusableFlowXmlTemplate(resource2.getPath(), connectionInfo);
    Assert.assertTrue(template2.isSuccess());
    Assert.assertFalse(template2.isReusableFlowOutputPortConnectionsNeeded());
    // get the flow for the parent processor and verify it is connected to the other reusable flow
    NifiFlowProcessGroup flow = getFlow(template2.getTemplateResults().getProcessGroupEntity().getId());
    Assert.assertNotNull(flow);
    boolean testUpdate2ProcessorExists = flow.getProcessorMap().values().stream().anyMatch(p -> "test-update2".equalsIgnoreCase(p.getName()));
    Assert.assertTrue(testUpdate2ProcessorExists);
    return new ConnectedTemplate(template1, template2);
}

Example 5 with NifiFlowProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.flow.NifiFlowProcessGroup in project kylo by Teradata.

the class TestFlowBuilder method testFlowBuilder.

@Test
public void testFlowBuilder() {
    // build a graph of processors
    NifiVisitableProcessGroup parent = new NifiVisitableProcessGroup(processGroupDTO());
    NifiVisitableProcessor processor1 = processor();
    NifiVisitableProcessor processor2 = processor();
    NifiVisitableProcessor processor3 = processor();
    NifiVisitableProcessor processor4 = processor();
    NifiVisitableProcessor processor5 = processor();
    NifiVisitableProcessor processor6 = processor();
    NifiVisitableProcessor processor7 = processor();
    // connect processors
    connect(processor1, processor2);
    connect(processor2, processor3);
    connect(processor2, processor4);
    connect(processor4, processor5);
    connect(processor5, processor6);
    connect(processor5, processor7);
    parent.getStartingProcessors().add(processor1);
    NifiFlowBuilder builder = new NifiFlowBuilder();
    NifiFlowProcessGroup group = builder.build(parent);
    Assert.assertTrue(group.getStartingProcessors().size() == 1);
    // assert processors 3,6,7 are ending/leaf processors
    Assert.assertTrue(group.getEndingProcessors().size() == 3);
}