Examples with NifiVisitableProcessor com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor used on opensource projects

Example 1 with NifiVisitableProcessor

use of com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor in project kylo by Teradata.

the class DefaultNiFiFlowVisitorClient method getProcessorsForFlow.

public Set<ProcessorDTO> getProcessorsForFlow(String processGroupId) throws NifiComponentNotFoundException {
    NifiVisitableProcessGroup group = getFlowOrder(processGroupId, null);
    Set<ProcessorDTO> processors = new HashSet<>();
    for (NifiVisitableProcessor p : group.getStartingProcessors()) {
        processors.addAll(p.getProcessors());
    }
    return processors;
}

Example 2 with NifiVisitableProcessor

use of com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor in project kylo by Teradata.

the class NifiConnectionOrderVisitor method getRemoteProcessGroupAsVisitableProcessor.

private NifiVisitableProcessor getRemoteProcessGroupAsVisitableProcessor(String processGroupId, String parentGroupId) {
    NifiVisitableProcessor processor = visitedProcessors.get(processGroupId);
    if (processor == null) {
        ProcessorDTO processorDTO = processorsMap.get(processGroupId);
        if (processorDTO == null) {
            RemoteProcessGroupDTO remoteProcessGroupDTO = getRemoteGroup(processGroupId);
            if (remoteProcessGroupDTO != null) {
                processorDTO = new ProcessorDTO();
                processorDTO.setType("NiFi.RemoteProcessGroup");
                processorDTO.setId(processGroupId);
                processorDTO.setParentGroupId(parentGroupId);
                processorDTO.setName(remoteProcessGroupDTO.getName());
                processorDTO.setConfig(new ProcessorConfigDTO());
                processorsMap.put(processGroupId, processorDTO);
            }
        }
        if (processorDTO != null) {
            processor = new NifiVisitableProcessor(processorDTO);
        }
    }
    return processor;
}

Example 3 with NifiVisitableProcessor

use of com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor in project kylo by Teradata.

the class NifiConnectionOrderVisitor method getSourceProcessors.

public List<NifiVisitableProcessor> getSourceProcessors(ConnectionDTO connection) {
    ConnectableDTO source = connection.getSource();
    List<NifiVisitableProcessor> sourceProcessors = new ArrayList<>();
    if (source != null) {
        if ("INPUT_PORT".equalsIgnoreCase(source.getType())) {
            NifiVisitableProcessGroup group = visitedProcessGroups.get(source.getGroupId());
            if (group == null) {
                group = processGroup;
            }
            NifiVisitableProcessGroup parent = visitedProcessGroups.get(group.getDto().getParentGroupId());
            // if the parent is null the parent is the starting process group
            if (parent == null) {
                parent = processGroup;
            }
            ConnectionDTO conn = parent.getConnectionMatchingDestinationId(source.getId());
            if (conn == null) {
                // if its null get it from the cache and process with that.
                conn = cache.getProcessGroupCache().values().stream().flatMap(g -> g.getContents().getConnections().stream()).filter(connectionDTO -> connectionDTO.getDestination().getId().equalsIgnoreCase(source.getId())).findFirst().orElse(null);
            }
            if (conn != null && conn != connection) {
                // get the processor whos source matches this connection Id
                sourceProcessors = getSourceProcessors(conn);
            // assign the inputPortProcessor == the the destination of this connection
            }
            List<NifiVisitableProcessor> inputProcessors = getDestinationProcessors(connection, false);
            if (inputProcessors != null) {
                for (NifiVisitableProcessor inputProcessor : inputProcessors) {
                    currentProcessGroup.addInputPortProcessor(source.getId(), inputProcessor);
                }
            }
        } else if ("OUTPUT_PORT".equals(source.getType())) {
            // get the sources group id then get the ending processor for that group
            NifiVisitableProcessGroup group = visitedProcessGroups.get(source.getGroupId());
            if (group != null) {
                Set<NifiVisitableProcessor> sources = group.getOutputPortProcessors(source.getId());
                if (sourceProcessors != null && sources != null) {
                    sourceProcessors.addAll(sources);
                }
            /*
                   If a process group is connected to another process group without any processors there will be no source processors
                    having sourceProcessors as null here is ok.
                    */
            }
        } else if ("FUNNEL".equalsIgnoreCase(source.getType())) {
            List<ConnectionDTO> passThroughConnections = NifiConnectionUtil.findConnectionsMatchingDestinationId(currentProcessGroup.getDto().getContents().getConnections(), connection.getSource().getId());
            if (passThroughConnections != null) {
                for (ConnectionDTO dto : passThroughConnections) {
                    ConnectionDTO newConnection = new ConnectionDTO();
                    newConnection.setSource(dto.getSource());
                    newConnection.setId(connection.getSource().getId());
                    newConnection.setDestination(connection.getDestination());
                    visitConnection(new NifiVisitableConnection(currentProcessGroup, newConnection));
                }
            }
        } else if ("PROCESSOR".equals(source.getType())) {
            NifiVisitableProcessor sourceProcessor = getConnectionProcessor(source.getGroupId(), source.getId());
            sourceProcessors.add(sourceProcessor);
        }
        for (NifiVisitableProcessor sourceProcessor : sourceProcessors) {
            sourceProcessor.addDestinationConnectionIdentifier(connection);
        }
    }
    return sourceProcessors;
}

Example 4 with NifiVisitableProcessor

use of com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor in project kylo by Teradata.

the class NifiConnectionOrderVisitor method getConnectionProcessor.

private NifiVisitableProcessor getConnectionProcessor(String groupId, String id) {
    NifiVisitableProcessor processor = visitedProcessors.get(id);
    if (processor == null) {
        if (!this.processorsMap.containsKey(id)) {
            // if the current group is not related to this processgroup then attempt to walk this processors processgroup
            try {
                log.debug("fetch ProcessGroup for getConnectionProcessor {} ", groupId);
                ProcessGroupDTO processGroupEntity = getGroup(groupId);
                if (processGroupEntity != null) {
                    ProcessorDTO processorDTO = NifiProcessUtil.findFirstProcessorsById(processGroupEntity.getContents().getProcessors(), id);
                    if (processorDTO != null) {
                        this.processorsMap.put(id, processorDTO);
                    }
                    if (processGroup.getDto().getId() != groupId && !visitedProcessGroups.containsKey(processGroupEntity.getId())) {
                        visitProcessGroup(new NifiVisitableProcessGroup(processGroupEntity));
                    }
                } else {
                    log.error("getConnectionProcessor error.  Unable to find Process Group for process group id: {}, and processor: {} {} ", groupId, id);
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        // 
        processor = visitedProcessors.get(id);
        if (processor == null) {
            processor = new NifiVisitableProcessor(this.processorsMap.get(id));
            // visit the group?
            processor.accept(this);
        }
    }
    return processor;
}

Example 5 with NifiVisitableProcessor

use of com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessor in project kylo by Teradata.

the class NifiConnectionOrderVisitor method getDestinationProcessors.

public List<NifiVisitableProcessor> getDestinationProcessors(ConnectionDTO connection, boolean getSource) {
    ConnectableDTO dest = connection.getDestination();
    List<NifiVisitableProcessor> destinationProcessors = new ArrayList<>();
    if (dest != null) {
        if ("INPUT_PORT".equalsIgnoreCase(dest.getType())) {
            boolean isNew = false;
            NifiVisitableProcessGroup group = visitedProcessGroups.get(dest.getGroupId());
            if (group == null) {
                group = fetchProcessGroup(dest.getGroupId());
            }
            ConnectionDTO conn = group.getConnectionMatchingSourceId(dest.getId());
            if (conn != null) {
                destinationProcessors = getDestinationProcessors(conn, getSource);
                if (getSource) {
                    List<NifiVisitableProcessor> outputProcessors = getSourceProcessors(connection);
                    if (outputProcessors != null) {
                        for (NifiVisitableProcessor outputProcessor : outputProcessors) {
                            // outputProcessor.setOutputPortId(dest.getId());
                            currentProcessGroup.addOutputPortProcessor(dest.getId(), outputProcessor);
                        }
                    }
                }
            }
        }
        if ("REMOTE_INPUT_PORT".equalsIgnoreCase(dest.getType())) {
            boolean isNew = false;
            // treat this like a Processor for the connection graph
            NifiVisitableProcessor processorDto = getRemoteProcessGroupAsVisitableProcessor(dest.getGroupId(), connection.getParentGroupId());
            destinationProcessors.add(processorDto);
        } else if ("OUTPUT_PORT".equals(dest.getType())) {
            boolean isNew = false;
            // get parent processgroup connection to input port
            NifiVisitableProcessGroup group = visitedProcessGroups.get(dest.getGroupId());
            if (group == null) {
                group = fetchProcessGroup(dest.getGroupId());
            }
            ConnectionDTO conn = group.getConnectionMatchingSourceId(dest.getId());
            if (conn == null) {
                conn = searchConnectionMatchingSource(group.getDto().getParentGroupId(), dest.getId());
            }
            if (conn != null) {
                // get the processor whos source matches this connection Id
                List<NifiVisitableProcessor> destinations = getDestinationProcessors(conn, getSource);
                if (destinations != null) {
                    destinationProcessors.addAll(destinations);
                }
                if (getSource) {
                    List<NifiVisitableProcessor> outputProcessors = getSourceProcessors(connection);
                    if (outputProcessors != null) {
                        for (NifiVisitableProcessor outputProcessor : outputProcessors) {
                            currentProcessGroup.addOutputPortProcessor(dest.getId(), outputProcessor);
                        }
                    }
                }
            }
        } else if ("FUNNEL".equals(dest.getType())) {
            List<ConnectionDTO> passThroughConnections = NifiConnectionUtil.findConnectionsMatchingSourceId(currentProcessGroup.getDto().getContents().getConnections(), connection.getDestination().getId());
            if (passThroughConnections != null) {
                for (ConnectionDTO dto : passThroughConnections) {
                    ConnectionDTO newConnection = new ConnectionDTO();
                    newConnection.setId(connection.getSource().getId());
                    newConnection.setSource(connection.getSource());
                    newConnection.setDestination(dto.getDestination());
                    List<NifiVisitableProcessor> destinations = getDestinationProcessors(newConnection, getSource);
                    if (destinations != null) {
                        // destinationProcessor.setOutputPortId(dest.getId());
                        destinationProcessors.addAll(destinations);
                    }
                }
            }
        } else if ("PROCESSOR".equals(dest.getType())) {
            NifiVisitableProcessor destinationProcessor = getConnectionProcessor(dest.getGroupId(), dest.getId());
            destinationProcessors.add(destinationProcessor);
        }
    }
    for (NifiVisitableProcessor destinationProcessor : destinationProcessors) {
        destinationProcessor.addSourceConnectionIdentifier(connection);
    }
    return destinationProcessors;
}